diff --git a/.gitattributes b/.gitattributes
index ecdb89ea39665e81b05028d850031743d878f189..e88e8c5b164f87e4d963419c498b84a18e54cb62 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -79,3 +79,59 @@ qo464509/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
 8511ppcj/episode_rosbags/episode_3_2025-01-01-11-49-26.bag filter=lfs diff=lfs merge=lfs -text
 8511ppcj/episode_rosbags/episode_2_2025-01-01-11-48-26.bag filter=lfs diff=lfs merge=lfs -text
 8511ppcj/episode_rosbags/episode_4_2025-01-01-11-50-21.bag filter=lfs diff=lfs merge=lfs -text
+8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4 filter=lfs diff=lfs merge=lfs -text
+8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4 filter=lfs diff=lfs merge=lfs -text
+8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4 filter=lfs diff=lfs merge=lfs -text
+8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/0_eval_0_28199d60beae23009631.mp4 filter=lfs diff=lfs merge=lfs -text
+8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4 filter=lfs diff=lfs merge=lfs -text
+8511ppcj/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+8511ppcj/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+8511ppcj/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+8511ppcj/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+8511ppcj/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4k07t1xl/episode_rosbags/episode_3_2025-01-01-11-38-39.bag filter=lfs diff=lfs merge=lfs -text
+4k07t1xl/episode_rosbags/episode_1_2025-01-01-11-36-40.bag filter=lfs diff=lfs merge=lfs -text
+hab6e0c2/episode_rosbags/episode_1_2024-12-31-22-42-42.bag filter=lfs diff=lfs merge=lfs -text
+7krw5tkq/episode_rosbags/episode_2_2025-01-01-11-31-40.bag filter=lfs diff=lfs merge=lfs -text
+hab6e0c2/episode_rosbags/episode_3_2024-12-31-22-44-16.bag filter=lfs diff=lfs merge=lfs -text
+hab6e0c2/episode_rosbags/episode_2_2024-12-31-22-43-30.bag filter=lfs diff=lfs merge=lfs -text
+hab6e0c2/episode_rosbags/episode_0_2024-12-31-22-41-43.bag filter=lfs diff=lfs merge=lfs -text
+4k07t1xl/episode_rosbags/episode_2_2025-01-01-11-37-37.bag filter=lfs diff=lfs merge=lfs -text
+hab6e0c2/episode_rosbags/episode_4_2024-12-31-22-45-00.bag filter=lfs diff=lfs merge=lfs -text
+7krw5tkq/episode_rosbags/episode_1_2025-01-01-11-30-40.bag filter=lfs diff=lfs merge=lfs -text
+9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4 filter=lfs diff=lfs merge=lfs -text
+9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4 filter=lfs diff=lfs merge=lfs -text
+9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4 filter=lfs diff=lfs merge=lfs -text
+9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4 filter=lfs diff=lfs merge=lfs -text
+4k07t1xl/episode_rosbags/episode_4_2025-01-01-11-39-47.bag filter=lfs diff=lfs merge=lfs -text
+9rj28b8z/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9rj28b8z/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9rj28b8z/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9rj28b8z/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9rj28b8z/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4 filter=lfs diff=lfs merge=lfs -text
+qo464509/episode_rosbags/episode_1_2025-01-01-10-53-45.bag filter=lfs diff=lfs merge=lfs -text
+gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/1_eval_1_db2109c40f910f358754.mp4 filter=lfs diff=lfs merge=lfs -text
+gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4 filter=lfs diff=lfs merge=lfs -text
+gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4 filter=lfs diff=lfs merge=lfs -text
+gwnrzhmx/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4 filter=lfs diff=lfs merge=lfs -text
+gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4 filter=lfs diff=lfs merge=lfs -text
+gwnrzhmx/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+gwnrzhmx/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+gwnrzhmx/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+gwnrzhmx/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+8511ppcj/episode_rosbags/episode_1_2025-01-01-11-47-39.bag filter=lfs diff=lfs merge=lfs -text
+7krw5tkq/episode_rosbags/episode_0_2025-01-01-11-29-41.bag filter=lfs diff=lfs merge=lfs -text
+4k07t1xl/episode_rosbags/episode_0_2025-01-01-11-35-41.bag filter=lfs diff=lfs merge=lfs -text
+6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4 filter=lfs diff=lfs merge=lfs -text
+6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4 filter=lfs diff=lfs merge=lfs -text
+6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/4_eval_4_6071852b701bfb891669.mp4 filter=lfs diff=lfs merge=lfs -text
+6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4 filter=lfs diff=lfs merge=lfs -text
+6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4 filter=lfs diff=lfs merge=lfs -text
+qo464509/episode_rosbags/episode_3_2025-01-01-10-55-17.bag filter=lfs diff=lfs merge=lfs -text
+6necfy24/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6necfy24/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6necfy24/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6necfy24/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6necfy24/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
diff --git a/4k07t1xl/episode_rosbags/episode_0_2025-01-01-11-35-41.bag b/4k07t1xl/episode_rosbags/episode_0_2025-01-01-11-35-41.bag
new file mode 100644
index 0000000000000000000000000000000000000000..6ec2951cf36fccf89c3a8a9767adf5084697f743
--- /dev/null
+++ b/4k07t1xl/episode_rosbags/episode_0_2025-01-01-11-35-41.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:480bd5abbeb4e31a4ce480434a1e3059f8b60906e1fd4884fa000c6128acb2e6
+size 1409989659
diff --git a/4k07t1xl/episode_rosbags/episode_1_2025-01-01-11-36-40.bag b/4k07t1xl/episode_rosbags/episode_1_2025-01-01-11-36-40.bag
new file mode 100644
index 0000000000000000000000000000000000000000..d606b8bbdb1afab77fa5a6d2ec3f59a67f6e5cd8
--- /dev/null
+++ b/4k07t1xl/episode_rosbags/episode_1_2025-01-01-11-36-40.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfd725f973b0a12f3392bc17bc62d20b0f5ea802fbbc36574544b22cf2a03600
+size 1411342568
diff --git a/4k07t1xl/episode_rosbags/episode_2_2025-01-01-11-37-37.bag b/4k07t1xl/episode_rosbags/episode_2_2025-01-01-11-37-37.bag
new file mode 100644
index 0000000000000000000000000000000000000000..e6cb47b4979320ba9da8771f41a702954000c664
--- /dev/null
+++ b/4k07t1xl/episode_rosbags/episode_2_2025-01-01-11-37-37.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f64f3cb1f78cd21819b98ab9f357308f4de0c2578df800a02dea6291fb74b546
+size 1414163525
diff --git a/4k07t1xl/episode_rosbags/episode_3_2025-01-01-11-38-39.bag b/4k07t1xl/episode_rosbags/episode_3_2025-01-01-11-38-39.bag
new file mode 100644
index 0000000000000000000000000000000000000000..6f25ab142c1c0e8c9cc8f903fdfadc474fdc1342
--- /dev/null
+++ b/4k07t1xl/episode_rosbags/episode_3_2025-01-01-11-38-39.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:534b73cc0c7b04a723415145404da3e2ee193ee4b706e07da159f3ee4f214627
+size 1413434270
diff --git a/4k07t1xl/episode_rosbags/episode_4_2025-01-01-11-39-47.bag b/4k07t1xl/episode_rosbags/episode_4_2025-01-01-11-39-47.bag
new file mode 100644
index 0000000000000000000000000000000000000000..47f305d3b1c871781519a1290ce46820d6296814
--- /dev/null
+++ b/4k07t1xl/episode_rosbags/episode_4_2025-01-01-11-39-47.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8e32b31908673b7e36a1d4556fdd1483475b96a0cedabcd6764039cf85f87e6
+size 1414175696
diff --git a/6necfy24/.hydra/config.yaml b/6necfy24/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe
--- /dev/null
+++ b/6necfy24/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1007_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/6necfy24/.hydra/hydra.yaml b/6necfy24/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f2a2ad4910fdd86d6d4dcf11e69f7437a5583593
--- /dev/null
+++ b/6necfy24/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/6necfy24/.hydra/overrides.yaml b/6necfy24/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/6necfy24/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/6necfy24/episode_rosbags/aligned_depth_to_color_K.npy b/6necfy24/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/6necfy24/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/6necfy24/episode_rosbags/cam_tf_world.npy b/6necfy24/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/6necfy24/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/6necfy24/episode_rosbags/color_K.npy b/6necfy24/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/6necfy24/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/6necfy24/episode_rosbags/depth_K.npy b/6necfy24/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/6necfy24/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/6necfy24/eval_robot.log b/6necfy24/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..605119b388795ccd2297870bf8823e87b58f9996
--- /dev/null
+++ b/6necfy24/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-01 10:57:36,006][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-01 10:57:36,009][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/6necfy24/eval_video/0_eval.mp4 b/6necfy24/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9520025a9ca97f8cf2dd63c59e86e4a385a3f73b
--- /dev/null
+++ b/6necfy24/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:259d97e9bef5a9deb16734cc2119456df9a5111ee59179b967dcaab68fd5754a
+size 1184839
diff --git a/6necfy24/eval_video/1_eval.mp4 b/6necfy24/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..981ca89d5e6e772eea085363d99d0308ccd22b89
--- /dev/null
+++ b/6necfy24/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09700a0ebedadd7f6657f1f5b14258fd54cdf763558190c9c81258f46e43bdec
+size 1180544
diff --git a/6necfy24/eval_video/2_eval.mp4 b/6necfy24/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..cabb3640541b116b81a664391dbc7baee95640ff
--- /dev/null
+++ b/6necfy24/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ac987b25093947b056a0fc3f009f1b1dc3653703a6e94a75e1e0117fb995bf1
+size 1125933
diff --git a/6necfy24/eval_video/3_eval.mp4 b/6necfy24/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..7405e72e82f0b7a3b6b767c820defff90d9ce09f
--- /dev/null
+++ b/6necfy24/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c141374c1432359d5cd74a7de924442270ee1ae4e9a2456033c63ae75c9a6e1
+size 1223483
diff --git a/6necfy24/eval_video/4_eval.mp4 b/6necfy24/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..0b919355ae6bb0ebded7e33dda0e525961ad6e1d
--- /dev/null
+++ b/6necfy24/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6071852b701bfb8916698bb10a1d5f49bbf55b0e18e3637612b9a8e9bf261448
+size 1183242
diff --git a/6necfy24/tb/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0 b/6necfy24/tb/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
new file mode 100644
index 0000000000000000000000000000000000000000..7242b58fc2d067ea6a3512eb5558a9550b12eb4b
--- /dev/null
+++ b/6necfy24/tb/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c017c2abf7eed609a8c1f2a968029deb729a91851ee882363cbf7eb14f465eb
+size 6539
diff --git a/6necfy24/wandb/debug-internal.log b/6necfy24/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..dbbd5a333a459ad34cba7a67ff89c22b4f3064a7
--- /dev/null
+++ b/6necfy24/wandb/debug-internal.log
@@ -0,0 +1,779 @@
+2025-01-01 10:57:41,755 INFO    StreamThr :1788056 [internal.py:wandb_internal():85] W&B internal server running at pid: 1788056, started at: 2025-01-01 10:57:41.755427
+2025-01-01 10:57:41,756 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status
+2025-01-01 10:57:41,768 INFO    WriterThread:1788056 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/run-6necfy24.wandb
+2025-01-01 10:57:41,769 DEBUG   SenderThread:1788056 [sender.py:send():379] send: header
+2025-01-01 10:57:41,778 DEBUG   SenderThread:1788056 [sender.py:send():379] send: run
+2025-01-01 10:57:41,983 INFO    SenderThread:1788056 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files
+2025-01-01 10:57:41,983 INFO    SenderThread:1788056 [sender.py:_start_run_threads():1188] run started: 6necfy24 with start time 1735747061.75616
+2025-01-01 10:57:41,999 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: check_version
+2025-01-01 10:57:41,999 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: check_version
+2025-01-01 10:57:42,043 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: run_start
+2025-01-01 10:57:42,048 DEBUG   HandlerThread:1788056 [system_info.py:__init__():26] System info init
+2025-01-01 10:57:42,048 DEBUG   HandlerThread:1788056 [system_info.py:__init__():41] System info init done
+2025-01-01 10:57:42,048 INFO    HandlerThread:1788056 [system_monitor.py:start():194] Starting system monitor
+2025-01-01 10:57:42,048 INFO    SystemMonitor:1788056 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-01 10:57:42,049 INFO    HandlerThread:1788056 [system_monitor.py:probe():214] Collecting system info
+2025-01-01 10:57:42,049 INFO    SystemMonitor:1788056 [interfaces.py:start():188] Started cpu monitoring
+2025-01-01 10:57:42,050 INFO    SystemMonitor:1788056 [interfaces.py:start():188] Started disk monitoring
+2025-01-01 10:57:42,050 INFO    SystemMonitor:1788056 [interfaces.py:start():188] Started gpu monitoring
+2025-01-01 10:57:42,051 INFO    SystemMonitor:1788056 [interfaces.py:start():188] Started memory monitoring
+2025-01-01 10:57:42,051 INFO    SystemMonitor:1788056 [interfaces.py:start():188] Started network monitoring
+2025-01-01 10:57:42,060 DEBUG   HandlerThread:1788056 [system_info.py:probe():152] Probing system
+2025-01-01 10:57:42,062 DEBUG   HandlerThread:1788056 [system_info.py:_probe_git():137] Probing git
+2025-01-01 10:57:42,066 DEBUG   HandlerThread:1788056 [system_info.py:_probe_git():145] Probing git done
+2025-01-01 10:57:42,066 DEBUG   HandlerThread:1788056 [system_info.py:probe():200] Probing system done
+2025-01-01 10:57:42,066 DEBUG   HandlerThread:1788056 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T15:57:42.060972', 'startedAt': '2025-01-01T15:57:41.743321', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.4958749999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 4500.008, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 657.4940643310547}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-01 10:57:42,066 INFO    HandlerThread:1788056 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-01 10:57:42,066 INFO    HandlerThread:1788056 [system_monitor.py:probe():227] Publishing system info
+2025-01-01 10:57:42,067 DEBUG   HandlerThread:1788056 [system_info.py:_save_code():44] Saving code
+2025-01-01 10:57:42,068 DEBUG   HandlerThread:1788056 [system_info.py:_save_code():67] Saving code done
+2025-01-01 10:57:42,068 DEBUG   HandlerThread:1788056 [system_info.py:_save_patches():84] Saving git patches
+2025-01-01 10:57:42,130 DEBUG   HandlerThread:1788056 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-01 10:57:42,131 INFO    HandlerThread:1788056 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-01 10:57:42,134 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 10:57:42,134 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-01 10:57:42,134 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-01 10:57:42,134 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-01 10:57:42,374 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-01 10:57:42,374 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: python_packages
+2025-01-01 10:57:42,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:57:42,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:57:42,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:42,379 INFO    HandlerThread:1788056 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/tb"
+  save: true
+}
+control {
+  relay_id: "e70ba2374c284bda9de162ecdf68005d"
+}
+_info {
+  stream_id: "6necfy24"
+}
+
+2025-01-01 10:57:42,379 DEBUG   HandlerThread:1788056 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-01 10:57:42,390 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/j1t6nnty-code/FISH/eval_robot.py
+2025-01-01 10:57:42,392 INFO    wandb-upload_0:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/kmz6eb0n-wandb-metadata.json
+2025-01-01 10:57:42,400 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/vzu4x605-diff.patch
+2025-01-01 10:57:42,506 DEBUG   SenderThread:1788056 [sender.py:send():379] send: telemetry
+2025-01-01 10:57:42,506 DEBUG   SenderThread:1788056 [sender.py:send():379] send: tbrecord
+2025-01-01 10:57:42,506 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 10:57:42,506 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0 with policy live
+2025-01-01 10:57:42,689 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/p0ywqfw0-events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:57:42,985 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/code/FISH/eval_robot.py
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/requirements.txt
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-metadata.json
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/diff.patch
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/code/FISH
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/code
+2025-01-01 10:57:43,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:44,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:44,984 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:45,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:46,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:47,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:57:47,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:48,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:49,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:49,447 DEBUG   SenderThread:1788056 [sender.py:send():379] send: config
+2025-01-01 10:57:50,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:50,985 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:51,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:52,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:57:52,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:52,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:53,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:54,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:54,987 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:55,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:56,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:56,987 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:57,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:57:57,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:57:57,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:57,464 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:57:58,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:58,988 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:59,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:00,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:01,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:02,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:03,253 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:03,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:04,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:04,990 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:05,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:06,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:06,990 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:07,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:08,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:08,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:08,991 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:09,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:10,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:11,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:12,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:58:12,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:58:12,417 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:13,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:13,445 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:13,992 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml
+2025-01-01 10:58:14,388 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:15,388 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:16,388 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:17,388 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:18,388 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:19,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:19,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:20,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:21,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:22,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:23,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:24,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:24,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:25,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:26,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:27,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:58:27,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:58:27,417 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:28,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:29,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:29,464 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:30,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:31,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:32,391 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:33,392 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:33,931 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 10:58:33,931 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4 with policy now
+2025-01-01 10:58:33,931 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 10:58:33,932 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 10:58:33,934 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 10:58:33,935 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 10:58:33,997 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4
+2025-01-01 10:58:33,998 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 10:58:33,998 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval
+2025-01-01 10:58:33,998 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos
+2025-01-01 10:58:33,998 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media
+2025-01-01 10:58:34,188 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/eyx99nu8-media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4
+2025-01-01 10:58:34,392 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:34,449 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/0abzwge7-media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4
+2025-01-01 10:58:34,942 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:34,998 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:35,391 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:36,393 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:37,394 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:38,393 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:39,394 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:40,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:40,493 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:41,000 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:41,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:42,051 DEBUG   SystemMonitor:1788056 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-01 10:58:42,052 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 10:58:42,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:58:42,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:58:42,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:43,000 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:43,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:44,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:45,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:46,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:46,491 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:47,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:48,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:49,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:50,003 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:58:50,237 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/djkmibn0-events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:58:50,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:51,003 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:51,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:51,817 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:52,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:53,004 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:53,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:54,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:55,005 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:55,397 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:56,397 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:57,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:58:57,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:58:57,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:57,527 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:58,398 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:59,398 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:00,398 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:01,398 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:02,398 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:03,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:03,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:04,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:05,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:06,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:07,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:08,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:08,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:09,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:10,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:11,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:12,053 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 10:59:12,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:59:12,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:59:12,417 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:13,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:13,440 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:14,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:15,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:16,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:17,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:18,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:19,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:19,404 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:20,404 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:21,408 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:21,465 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 10:59:21,465 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 10:59:21,465 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4 with policy now
+2025-01-01 10:59:21,473 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 10:59:21,473 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 10:59:21,475 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 10:59:21,718 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/rhw6z3fz-media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4
+2025-01-01 10:59:22,013 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 10:59:22,013 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4
+2025-01-01 10:59:22,013 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval
+2025-01-01 10:59:22,301 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/thheyvvn-media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4
+2025-01-01 10:59:22,409 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:23,013 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:59:23,405 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:24,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:24,411 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:25,415 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:26,417 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:27,015 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:59:27,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:59:27,378 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:59:27,425 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:28,410 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:29,015 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:59:29,410 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:29,982 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:30,412 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:31,415 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:32,410 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:33,410 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:34,410 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:35,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:35,411 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:36,412 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:37,411 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:38,411 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:39,411 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:40,412 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:41,380 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:41,412 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:42,053 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 10:59:42,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:59:42,377 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:59:42,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:43,413 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:44,413 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:45,413 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:46,414 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:46,469 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:47,414 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:48,414 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:49,415 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:50,416 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:51,415 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:52,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:52,414 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:52,812 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 10:59:52,812 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 10:59:52,812 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 10:59:53,024 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:59:53,024 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 10:59:53,267 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/2klaftrg-events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:59:53,415 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:54,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:55,025 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:59:55,419 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:56,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:57,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:59:57,376 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:59:57,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:57,481 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:58,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:59,026 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:59:59,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:00,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:01,027 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:01,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:02,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:03,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:03,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:04,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:05,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:06,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:07,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:08,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:09,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:09,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:10,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:11,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:12,053 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:00:12,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:00:12,377 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:00:12,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:13,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:14,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:14,459 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:15,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:16,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:17,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:18,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:19,423 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:19,917 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:20,423 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:21,034 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:21,423 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:22,423 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:23,437 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:24,435 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:24,745 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 11:00:24,746 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4 with policy now
+2025-01-01 11:00:24,750 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:00:24,750 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:00:24,750 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:00:24,750 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:00:24,991 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/yj2aawlf-media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4
+2025-01-01 11:00:25,035 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:25,036 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:00:25,036 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4
+2025-01-01 11:00:25,036 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval
+2025-01-01 11:00:25,315 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/gq70ldhr-media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4
+2025-01-01 11:00:25,436 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:25,769 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:26,436 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:27,036 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:27,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:00:27,379 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:00:27,438 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:28,437 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:29,440 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:30,438 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:30,825 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:31,038 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:31,441 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:32,441 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:33,441 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:34,441 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:35,039 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:35,440 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:36,213 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:36,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:37,040 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:37,447 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:38,447 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:38,647 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:00:38,647 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:00:38,647 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:00:39,040 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:00:39,040 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:00:39,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:40,453 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:41,041 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:41,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:41,613 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:42,054 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:00:42,397 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:00:42,398 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:00:42,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:43,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:44,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:45,042 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:45,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:46,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:46,850 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:47,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:48,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:49,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:50,450 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:51,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:52,385 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:52,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:53,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:54,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:55,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:56,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:57,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:00:57,378 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:00:57,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:57,503 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:58,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:59,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:00,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:01,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:02,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:03,385 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:03,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:04,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:05,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:06,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:07,453 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:08,385 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:08,453 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:09,453 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:10,453 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:11,457 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:11,984 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 11:01:11,985 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4 with policy now
+2025-01-01 11:01:11,988 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:01:11,989 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:01:11,989 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:01:11,990 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:01:12,051 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:01:12,051 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4
+2025-01-01 11:01:12,052 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval
+2025-01-01 11:01:12,054 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:01:12,241 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/9u3y1704-media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4
+2025-01-01 11:01:12,384 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:01:12,384 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:01:12,459 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:12,500 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/3nl4yisw-media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4
+2025-01-01 11:01:13,052 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:13,464 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:13,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:14,475 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:15,480 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:16,469 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:17,479 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:18,468 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:19,054 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:19,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:19,463 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:20,466 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:21,054 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:21,464 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:22,473 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:23,468 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:24,391 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:24,466 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:25,467 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:26,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:27,386 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:01:27,386 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:01:27,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:28,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:29,057 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:01:29,293 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/mlsq6ykk-events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:01:29,479 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:29,708 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:01:29,709 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:01:29,709 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:29,709 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:01:30,057 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:01:30,479 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:31,058 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:31,473 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:32,474 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:33,058 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:33,469 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:34,469 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:34,929 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:35,059 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:35,469 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:36,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:37,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:38,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:39,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:40,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:40,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:41,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:42,055 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:01:42,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:01:42,382 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:01:42,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:43,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:44,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:45,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:45,525 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:46,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:47,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:48,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:49,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:50,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:51,391 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:51,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:52,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:53,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:54,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:55,066 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:55,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:56,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:56,910 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:57,382 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:01:57,382 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:01:57,473 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:58,473 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:59,473 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:00,477 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:01,068 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:01,203 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 11:02:01,203 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6071852b701bfb891669.mp4 with policy now
+2025-01-01 11:02:01,205 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:02:01,206 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:02:01,206 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:01,206 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:01,458 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/mby126yl-media/videos/eval/4_eval_4_6071852b701bfb891669.mp4
+2025-01-01 11:02:01,474 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:02,069 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:02,069 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/4_eval_4_6071852b701bfb891669.mp4
+2025-01-01 11:02:02,069 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval
+2025-01-01 11:02:02,238 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:02,369 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/d2f2eaxf-media/videos/eval/4_eval_4_6071852b701bfb891669.mp4
+2025-01-01 11:02:02,486 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:03,069 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:03,483 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:04,484 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:05,490 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:06,485 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:07,071 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:07,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:07,494 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:08,494 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:09,497 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:10,499 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:11,072 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:11,498 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:12,055 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:02:12,397 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:02:12,398 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:02:12,496 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:12,499 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:13,496 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:14,498 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:15,499 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:16,503 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:17,513 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:17,654 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-01 11:02:17,654 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: log_artifact
+2025-01-01 11:02:17,832 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:02:18,074 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:02:18,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json
+2025-01-01 11:02:18,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media
+2025-01-01 11:02:18,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table
+2025-01-01 11:02:18,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval
+2025-01-01 11:02:18,124 INFO    wandb-upload_1:1788056 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd7jhhpw5
+2025-01-01 11:02:18,236 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-01 11:02:18,244 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:02:18,286 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/qpqsfr8z-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json
+2025-01-01 11:02:18,497 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:18,598 INFO    SenderThread:1788056 [sender.py:send_request_log_artifact():1525] logged artifact run-6necfy24-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY3MDEzNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTIzOTU4MQ==', 'latestArtifact': None}}
+2025-01-01 11:02:18,598 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 11:02:18,598 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now
+2025-01-01 11:02:18,598 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:18,599 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:02:18,599 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,599 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,600 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:02:18,600 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,603 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,604 DEBUG   SenderThread:1788056 [sender.py:send():379] send: telemetry
+2025-01-01 11:02:18,604 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,604 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,604 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,604 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,605 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,605 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,605 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,605 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,606 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,606 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,606 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,606 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,607 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,607 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,607 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,607 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,608 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,608 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,608 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,608 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,609 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,609 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,609 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: log_artifact
+2025-01-01 11:02:18,846 INFO    wandb-upload_0:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/pc4x4hed-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json
+2025-01-01 11:02:18,959 INFO    wandb-upload_2:1788056 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpktagczqq
+2025-01-01 11:02:19,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:02:19,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:19,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:19,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json
+2025-01-01 11:02:19,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval
+2025-01-01 11:02:19,339 INFO    SenderThread:1788056 [sender.py:send_request_log_artifact():1525] logged artifact run-6necfy24-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY3MDE0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTIzOTU4NA==', 'latestArtifact': None}}
+2025-01-01 11:02:19,340 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 11:02:19,340 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json with policy now
+2025-01-01 11:02:19,340 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:02:19,340 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:19,341 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:19,497 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:19,518 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/712cyue4-media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json
+2025-01-01 11:02:19,774 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/1xs3cf6p-media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json
+2025-01-01 11:02:20,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:20,497 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:20,609 DEBUG   SenderThread:1788056 [sender.py:send():379] send: exit
+2025-01-01 11:02:20,609 INFO    SenderThread:1788056 [sender.py:send_exit():586] handling exit code: 0
+2025-01-01 11:02:20,610 INFO    SenderThread:1788056 [sender.py:send_exit():588] handling runtime: 278
+2025-01-01 11:02:20,610 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:20,610 INFO    SenderThread:1788056 [sender.py:send_exit():594] send defer
+2025-01-01 11:02:20,610 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:20,610 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-01 11:02:20,610 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:20,610 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-01 11:02:20,610 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 1
+2025-01-01 11:02:20,611 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:20,611 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-01 11:02:20,611 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:20,611 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-01 11:02:20,611 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 2
+2025-01-01 11:02:20,611 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:20,611 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-01 11:02:20,611 INFO    HandlerThread:1788056 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-01 11:02:20,611 INFO    HandlerThread:1788056 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-01 11:02:20,612 DEBUG   SystemMonitor:1788056 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-01 11:02:20,612 DEBUG   SystemMonitor:1788056 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-01 11:02:20,612 INFO    HandlerThread:1788056 [interfaces.py:finish():200] Joined disk monitor
+2025-01-01 11:02:20,663 INFO    HandlerThread:1788056 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-01 11:02:20,664 INFO    HandlerThread:1788056 [interfaces.py:finish():200] Joined memory monitor
+2025-01-01 11:02:20,664 INFO    HandlerThread:1788056 [interfaces.py:finish():200] Joined network monitor
+2025-01-01 11:02:20,664 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:20,664 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-01 11:02:20,664 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 3
+2025-01-01 11:02:20,664 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:02:20,664 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:20,665 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-01 11:02:20,665 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:20,665 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-01 11:02:20,665 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 4
+2025-01-01 11:02:20,665 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:20,665 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-01 11:02:21,076 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:21,076 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:23,077 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:26,771 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:02:26,771 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:26,771 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-01 11:02:26,772 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:26,772 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-01 11:02:26,772 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 5
+2025-01-01 11:02:26,772 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:02:26,773 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:26,773 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:02:26,773 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-01 11:02:26,773 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:26,774 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:26,774 DEBUG   SenderThread:1788056 [sender.py:send():379] send: summary
+2025-01-01 11:02:26,774 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:26,774 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:26,775 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-01 11:02:26,775 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 6
+2025-01-01 11:02:26,775 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:26,775 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-01 11:02:26,775 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:26,775 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-01 11:02:26,778 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:26,878 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 7
+2025-01-01 11:02:26,878 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:26,878 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-01 11:02:26,879 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:26,879 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-01 11:02:27,078 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml
+2025-01-01 11:02:27,078 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:27,610 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:02:30,743 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 8
+2025-01-01 11:02:30,743 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:02:30,744 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:30,744 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-01 11:02:30,744 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:30,744 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-01 11:02:30,744 INFO    SenderThread:1788056 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-01 11:02:30,746 INFO    SenderThread:1788056 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-01 11:02:30,748 INFO    SenderThread:1788056 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-01 11:02:30,757 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 9
+2025-01-01 11:02:30,757 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:30,758 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-01 11:02:30,758 DEBUG   SenderThread:1788056 [sender.py:send():379] send: artifact
+2025-01-01 11:02:30,936 INFO    SenderThread:1788056 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}}
+2025-01-01 11:02:30,937 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:30,937 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-01 11:02:30,937 INFO    SenderThread:1788056 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-01 11:02:31,079 INFO    SenderThread:1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:31,079 INFO    SenderThread:1788056 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml config.yaml
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json wandb-summary.json
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0 events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/requirements.txt requirements.txt
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-metadata.json wandb-metadata.json
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log output.log
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/diff.patch diff.patch
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-01 11:02:31,084 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/4_eval_4_6071852b701bfb891669.mp4 media/videos/eval/4_eval_4_6071852b701bfb891669.mp4
+2025-01-01 11:02:31,084 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4 media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4
+2025-01-01 11:02:31,084 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4 media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4
+2025-01-01 11:02:31,084 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4 media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4
+2025-01-01 11:02:31,084 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4 media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4
+2025-01-01 11:02:31,086 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json
+2025-01-01 11:02:31,086 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json
+2025-01-01 11:02:31,086 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 10
+2025-01-01 11:02:31,087 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:31,089 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-01 11:02:31,089 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:31,089 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-01 11:02:31,089 INFO    SenderThread:1788056 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-01 11:02:31,263 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml
+2025-01-01 11:02:31,283 INFO    wandb-upload_0:1788056 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:31,311 INFO    wandb-upload_4:1788056 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:31,321 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/831ywcq5-events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:02:31,338 INFO    wandb-upload_3:1788056 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/requirements.txt
+2025-01-01 11:02:31,538 INFO    Thread-11 (_thread_body):1788056 [sender.py:transition_state():614] send defer: 11
+2025-01-01 11:02:31,538 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:31,538 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-01 11:02:31,539 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:31,539 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-01 11:02:31,539 INFO    SenderThread:1788056 [file_pusher.py:join():175] waiting for file pusher
+2025-01-01 11:02:31,540 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 12
+2025-01-01 11:02:31,540 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:31,540 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-01 11:02:31,540 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:31,541 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-01 11:02:31,541 INFO    SenderThread:1788056 [file_stream.py:finish():601] file stream finish called
+2025-01-01 11:02:31,601 INFO    SenderThread:1788056 [file_stream.py:finish():605] file stream finish is done
+2025-01-01 11:02:31,602 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 13
+2025-01-01 11:02:31,602 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:31,602 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-01 11:02:31,602 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:31,602 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-01 11:02:31,602 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 14
+2025-01-01 11:02:31,602 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:31,602 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-01 11:02:31,603 DEBUG   SenderThread:1788056 [sender.py:send():379] send: final
+2025-01-01 11:02:31,603 DEBUG   SenderThread:1788056 [sender.py:send():379] send: footer
+2025-01-01 11:02:31,603 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:31,603 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-01 11:02:31,603 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:02:31,604 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: server_info
+2025-01-01 11:02:31,604 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:02:31,604 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:02:31,604 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: server_info
+2025-01-01 11:02:31,606 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-01 11:02:31,606 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-01 11:02:31,607 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:31,658 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:02:31,658 INFO    MainThread:1788056 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-01 11:02:31,659 INFO    MainThread:1788056 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-01 11:02:31,659 INFO    MainThread:1788056 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-01 11:02:31,659 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-01 11:02:31,659 INFO    HandlerThread:1788056 [handler.py:finish():882] shutting down handler
+2025-01-01 11:02:32,605 INFO    WriterThread:1788056 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/run-6necfy24.wandb
+2025-01-01 11:02:32,658 INFO    SenderThread:1788056 [sender.py:finish():1615] shutting down sender
+2025-01-01 11:02:32,658 INFO    SenderThread:1788056 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-01 11:02:32,658 INFO    SenderThread:1788056 [file_pusher.py:join():175] waiting for file pusher
diff --git a/6necfy24/wandb/debug.log b/6necfy24/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..8fea950f0879fbe189fa3d451e97050b4bfca392
--- /dev/null
+++ b/6necfy24/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-01 10:57:41,749 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Configure stats pid to 1787880
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/settings
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/logs/debug.log
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/logs/debug-internal.log
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:init():569] calling init triggers
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24'}
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:init():619] starting backend
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:init():623] setting up manager
+2025-01-01 10:57:41,754 INFO    MainThread:1787880 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-01 10:57:41,756 INFO    MainThread:1787880 [wandb_init.py:init():631] backend started and connected
+2025-01-01 10:57:41,768 INFO    MainThread:1787880 [wandb_init.py:init():720] updated telemetry
+2025-01-01 10:57:41,775 INFO    MainThread:1787880 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-01 10:57:41,998 INFO    MainThread:1787880 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-01 10:57:42,039 INFO    MainThread:1787880 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-01 10:57:42,039 INFO    MainThread:1787880 [wandb_init.py:init():804] starting run threads in backend
+2025-01-01 10:57:42,375 INFO    MainThread:1787880 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-01 10:57:42,376 INFO    MainThread:1787880 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-01 10:57:42,376 INFO    MainThread:1787880 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-01 10:57:42,376 INFO    MainThread:1787880 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-01 10:57:42,378 INFO    MainThread:1787880 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-01 10:57:42,379 INFO    MainThread:1787880 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/tb, True
+2025-01-01 10:57:49,446 INFO    MainThread:1787880 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'}
+2025-01-01 11:02:32,660 WARNING MsgRouterThr:1787880 [router.py:message_loop():77] message_loop has been closed
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/code/FISH/eval_robot.py b/6necfy24/wandb/run-20250101_105741-6necfy24/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml b/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4144314a3252ff8df077ab6203bb76796feed264
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1007_0'
+wandb_notes:
+  desc: null
+  value: 1007_0_restarted_2
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1735747061
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: fowlers
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/diff.patch b/6necfy24/wandb/run-20250101_105741-6necfy24/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..eff8de3c864ce1e6f5e6f3000cbf990769adf78e
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/diff.patch
@@ -0,0 +1,105 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 0f6a9ec..b9e3dfe 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
++# wandb_run_id: '1002_0' 
++
++# all books, 6/20 demos per book
++# RGBD+mask+act history
+ wandb_run_id: '1007_0' 
+ 
++# RGBD+mask+contact(all ftrs)+act history
++# wandb_run_id: '1006_0'
++
++# RGBD+mask+contact(contact+DTC)+act history
++# wandb_run_id: '1745_0'
+ 
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 21e335c..71a5d1c 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -155,7 +155,12 @@ run_id_list = [
+     # '1022_0',
+     # '1023_0',
+     # '1007_0',
+-    '1007_0'
++    # '1007_0'
++]
++
++run_id_list = [
++    # '1006_0',
++    '1745_0'
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py
+index 4179186..50493fe 100644
+--- a/FISH/eval_robot.py
++++ b/FISH/eval_robot.py
+@@ -478,8 +478,8 @@ class Workspace:
+ 				"summary_metrics.episode": {"$gte": 5},
+ 				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+ 				"state": "finished",
+-				"config.grasped_obj_name": grasped_obj_name,
+-				"config.left_book_slot": left_book_slot,
++				# "config.grasped_obj_name": grasped_obj_name,
++				# "config.left_book_slot": left_book_slot,
+ 			}
+ 
+ 			api = wandb.Api()
+@@ -490,7 +490,8 @@ class Workspace:
+ 			if len(filtered_runs) > 0:
+ 				for filtered_run in filtered_runs:
+ 					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+-					total_num_episodes += filtered_run.summary_metrics['episode']
++					# total_num_episodes += filtered_run.summary_metrics['episode']
++					total_num_episodes += filtered_run.config['num_eval']
+ 					list_of_historical_run_ids.append(filtered_run.id)
+ 
+ 			wandb.summary['total_num_successes'] = total_num_successes
+@@ -513,7 +514,6 @@ class Workspace:
+ 				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+ 				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+ 
+-
+ 			total_table_data = [total_table_data]
+ 			wandb.log({
+ 				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py
+index 3302c31..1946eca 100644
+--- a/gym-envs/gym_envs/envs/franka_env.py
++++ b/gym-envs/gym_envs/envs/franka_env.py
+@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env):
+ 
+ 	def mask_obs_callback(self, msg):
+ 		msg = self.possibly_resize_mask_obs(msg)
+-		rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}")
++		# rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}")
+ 		self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg)  
+ 
+ 	def contact_obs_callback(self, msg):
+diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py
+index e3031af..6f97e26 100644
+--- a/gym-envs/gym_envs/envs/hardware/franka.py
++++ b/gym-envs/gym_envs/envs/hardware/franka.py
+@@ -330,11 +330,11 @@ class Franka:
+ 		# self.yaw = yaw
+ 
+ 		self.move_to_target_hz = 10.0
+-		self.move_to_trans_tolerance = 0.01
+-		self.move_to_rot_tolerance = 2.0
++		self.move_to_trans_tolerance = 0.005
++		self.move_to_rot_tolerance = 1.0
+ 		self.move_to_target_max_velocity = 0.8
+ 		self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz)
+-		self.move_to_target_timeout = 15.0
++		self.move_to_target_timeout = 5.0
+ 		self.move_to_target_max_rotation = 25.0
+ 		# only can use one of these at a time
+ 		# assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw"
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..2f85f61c92088a8993fdf1bc1b36967832fa135d
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.25744009022795533, 0.9953789374731651, 0.284928768916229, 0.987118244073719, 0.4244327843340374, 0.9746108054165524, 0.6104677766340574, 0.9433402008421657]]}
\ No newline at end of file
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..45a5b8ca4b47c4a70ff2267bb32c0748a50c506a
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[23, 40, 0.575, 0.387843728087922, 0.7461782246335723, 0.424182772651577, 0.7285965382595633, 0.44806925208415094, 0.694331735333677, 0.46582129970876485, 0.6801309808728445]]}
\ No newline at end of file
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4 b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9520025a9ca97f8cf2dd63c59e86e4a385a3f73b
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:259d97e9bef5a9deb16734cc2119456df9a5111ee59179b967dcaab68fd5754a
+size 1184839
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4 b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..981ca89d5e6e772eea085363d99d0308ccd22b89
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09700a0ebedadd7f6657f1f5b14258fd54cdf763558190c9c81258f46e43bdec
+size 1180544
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4 b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..cabb3640541b116b81a664391dbc7baee95640ff
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ac987b25093947b056a0fc3f009f1b1dc3653703a6e94a75e1e0117fb995bf1
+size 1125933
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4 b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..7405e72e82f0b7a3b6b767c820defff90d9ce09f
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c141374c1432359d5cd74a7de924442270ee1ae4e9a2456033c63ae75c9a6e1
+size 1223483
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/4_eval_4_6071852b701bfb891669.mp4 b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/4_eval_4_6071852b701bfb891669.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..0b919355ae6bb0ebded7e33dda0e525961ad6e1d
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/4_eval_4_6071852b701bfb891669.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6071852b701bfb8916698bb10a1d5f49bbf55b0e18e3637612b9a8e9bf261448
+size 1183242
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log b/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..e395c555bd08628d79ec6889d812102a241e2a5a
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
@@ -0,0 +1,279 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1735747069.447305]: resetting environment
+[INFO] [1735747069.450541]: cleared current plan
+[INFO] [1735747069.451582]: moving to home
+[INFO] [1735747070.453617]: reached home
+[INFO] [1735747070.454935]: reset action history
+[INFO] [1735747073.163687]: environment reset
+Starting episode 0
+[INFO] [1735747073.166992]: resetting environment
+[INFO] [1735747073.172024]: cleared current plan
+[INFO] [1735747073.172976]: moving to home
+[INFO] [1735747074.174599]: reached home
+[INFO] [1735747074.176309]: reset action history
+[INFO] [1735747076.883409]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1735747082.246747]: resetting environment
+[INFO] [1735747082.250188]: cleared current plan
+[INFO] [1735747082.250999]: moving to home
+[INFO] [1735747083.253278]: reached home
+[INFO] [1735747083.254842]: reset action history
+[INFO] [1735747085.963937]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/episode_rosbags/episode_0_2025-01-01-10-58-05.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1787880.log
+started roslaunch server http://158.130.50.37:40877/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [1788157]
+process[print_text-2]: started with pid [1788181]
+process[rosbag_record-3]: started with pid [1788182]
+started bagging!
+[INFO] [1735747112.984032]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1735747113.703711]: Stored episode 1.
+[INFO] [1735747113.705209]: Saving video...
+[INFO] [1735747113.911839]: Video saved!
+Starting episode 1
+[INFO] [1735747113.933834]: resetting environment
+[INFO] [1735747113.941427]: cleared current plan
+[INFO] [1735747113.941932]: moving to home
+[INFO] [1735747118.445690]: reached home
+[INFO] [1735747118.446134]: reset action history
+[INFO] [1735747121.157670]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735747129.771496]: resetting environment
+[INFO] [1735747129.773670]: cleared current plan
+[INFO] [1735747129.773831]: moving to home
+[INFO] [1735747130.775498]: reached home
+[INFO] [1735747130.775910]: reset action history
+[INFO] [1735747133.482936]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/episode_rosbags/episode_1_2025-01-01-10-58-53.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1787880.log
+started roslaunch server http://158.130.50.37:42989/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [1788291]
+process[print_text-5]: started with pid [1788292]
+process[rosbag_record-6]: started with pid [1788293]
+started bagging!
+[INFO] [1735747160.437307]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit[pub_text-4] killing on exit
+[INFO] [1735747161.207347]: Stored episode 2.
+[INFO] [1735747161.209405]: Saving video...
+[INFO] [1735747161.421276]: Video saved!
+Starting episode 2
+[INFO] [1735747161.451819]: resetting environment
+[INFO] [1735747161.467068]: cleared current plan
+[INFO] [1735747161.475514]: moving to home
+[INFO] [1735747165.183076]: reached home
+[INFO] [1735747165.190937]: reset action history
+[INFO] [1735747167.936109]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735747192.578406]: resetting environment
+[INFO] [1735747192.584439]: cleared current plan
+[INFO] [1735747192.584823]: moving to home
+[INFO] [1735747193.587388]: reached home
+[INFO] [1735747193.587873]: reset action history
+[INFO] [1735747196.296061]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/episode_rosbags/episode_2_2025-01-01-10-59-56.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1787880.log
+started roslaunch server http://158.130.50.37:41643/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [1788480]
+process[print_text-8]: started with pid [1788481]
+process[rosbag_record-9]: started with pid [1788482]
+started bagging!
+For topic gripper_width: timestamp difference is 100870961 for nearest: 1735747198878646010 - target: 1735747198777775049 at idx 37
+For topic gripper_width: timestamp difference is -116250167 for nearest: 1735747218678642621 - target: 1735747218794892788 at idx 249
+[INFO] [1735747223.250876]: Storing episode...
+[rosbag_record-9] killing on exit
+[pub_text-7] killing on exit
+[print_text-8] killing on exit
+[INFO] [1735747224.023807]: Stored episode 3.
+[INFO] [1735747224.030075]: Saving video...
+[WARN] [1735747224.602558]: Plan exhausted
+[WARN] [1735747224.632160]: Plan exhausted
+[WARN] [1735747224.670369]: Plan exhausted
+[WARN] [1735747224.713631]: Plan exhausted
+[WARN] [1735747224.749488]: Plan exhausted
+[INFO] [1735747224.708339]: Video saved!
+Starting episode 3
+[INFO] [1735747224.754331]: resetting environment
+[INFO] [1735747224.764253]: cleared current plan
+[INFO] [1735747224.769491]: moving to home
+Move to target timeout: 5.002378940582275. Trans error: 0.005757261998951435. Rot error: 0.8876788974735654
+[INFO] [1735747232.373646]: reached home
+[INFO] [1735747232.376995]: reset action history
+[INFO] [1735747235.166202]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735747238.548946]: resetting environment
+[INFO] [1735747238.563389]: cleared current plan
+[INFO] [1735747238.567420]: moving to home
+[INFO] [1735747239.570265]: reached home
+[INFO] [1735747239.571106]: reset action history
+[INFO] [1735747242.350743]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/episode_rosbags/episode_3_2025-01-01-11-00-42.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1787880.log
+started roslaunch server http://158.130.50.37:34359/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [1788652]
+process[print_text-11]: started with pid [1788653]
+process[rosbag_record-12]: started with pid [1788654]
+started bagging!
+[WARN] [1735747271.910049]: Plan exhausted
+[WARN] [1735747271.961315]: Plan exhausted
+[WARN] [1735747271.990562]: Plan exhausted
+[INFO] [1735747270.571275]: Storing episode...
+[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[rosbag_record-12] killing on exit
+[INFO] [1735747271.321958]: Stored episode 4.
+[INFO] [1735747271.325428]: Saving video...
+[INFO] [1735747271.948370]: Video saved!
+Starting episode 4
+[INFO] [1735747271.994943]: resetting environment
+[INFO] [1735747272.018839]: cleared current plan
+[INFO] [1735747272.020076]: moving to home
+[INFO] [1735747276.529645]: reached home
+[INFO] [1735747276.538598]: reset action history
+[INFO] [1735747279.374243]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735747288.821347]: resetting environment
+[INFO] [1735747288.833389]: cleared current plan
+[INFO] [1735747288.834379]: moving to home
+[INFO] [1735747289.836424]: reached home
+[INFO] [1735747289.846202]: reset action history
+[INFO] [1735747292.631703]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/episode_rosbags/episode_4_2025-01-01-11-01-32.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1787880.log
+started roslaunch server http://158.130.50.37:43955/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [1788776]
+process[print_text-14]: started with pid [1788777]
+process[rosbag_record-15]: started with pid [1788801]
+started bagging!
+For topic gripper_width: timestamp difference is -114645568 for nearest: 1735747313678641471 - target: 1735747313793287039 at idx 249
+[INFO] [1735747319.655681]: Storing episode...
+[pub_text-13] killing on exit[print_text-14] killing on exit
+[rosbag_record-15] killing on exit
+[INFO] [1735747320.497338]: Stored episode 5.
+[INFO] [1735747320.501558]: Saving video...
+[WARN] [1735747320.989771]: Plan exhausted
+[WARN] [1735747321.030034]: Plan exhausted
+[WARN] [1735747321.069666]: Plan exhausted
+[WARN] [1735747321.121098]: Plan exhausted
+[WARN] [1735747321.149477]: Plan exhausted
+[WARN] [1735747321.190051]: Plan exhausted
+[WARN] [1735747321.229950]: Plan exhausted
+[INFO] [1735747321.165836]: Video saved!
+[INFO] [1735747321.205752]: resetting environment
+[INFO] [1735747321.237315]: cleared current plan
+[INFO] [1735747321.237983]: moving to home
+[INFO] [1735747325.843009]: reached home
+[INFO] [1735747325.860176]: reset action history
+[INFO] [1735747328.728686]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/requirements.txt b/6necfy24/wandb/run-20250101_105741-6necfy24/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-metadata.json b/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..b3315617ad77639bcee1741798591cfc8c5c7529
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-01T15:57:42.060972",
+    "startedAt": "2025-01-01T15:57:41.743321",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3562.4958749999996,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.998,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.008,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 657.4940643310547
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json b/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..56092bb80ce255412bec35ad2f0d259b4e5433fd
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "259d97e9bef5a9deb16734cc2119456df9a5111ee59179b967dcaab68fd5754a", "size": 1184839, "path": "media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4"}, "global_step": 665, "_timestamp": 1735747338.1639137, "_runtime": 276.4077537059784, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "09700a0ebedadd7f6657f1f5b14258fd54cdf763558190c9c81258f46e43bdec", "size": 1180544, "path": "media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4"}, "eval/num_success": 4.0, "episode": 40.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.25744009017944336, "eval/success_rate_ub_0.01": 0.9953789114952087, "eval/success_rate_lb_0.025": 0.2849287688732147, "eval/success_rate_ub_0.025": 0.9871182441711426, "eval/success_rate_lb_0.05": 0.42443278431892395, "eval/success_rate_ub_0.05": 0.9746108055114746, "eval/success_rate_lb_0.1": 0.610467791557312, "eval/success_rate_ub_0.1": 0.9433401823043823, "eval/2_eval": {"_type": "video-file", "sha256": "3ac987b25093947b056a0fc3f009f1b1dc3653703a6e94a75e1e0117fb995bf1", "size": 1125933, "path": "media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "1c141374c1432359d5cd74a7de924442270ee1ae4e9a2456033c63ae75c9a6e1", "size": 1223483, "path": "media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "6071852b701bfb8916698bb10a1d5f49bbf55b0e18e3637612b9a8e9bf261448", "size": 1183242, "path": "media/videos/eval/4_eval_4_6071852b701bfb891669.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "f6f07faa4a586efd9baa9e785d4017c5140aeeafdfeb020c33b4bb37a7d266af", "size": 397, "artifact_path": "wandb-client-artifact://gqvr4e3dunnz2ozo7cfkwb1l3s4gpqlwpzfjepcaxh9g8k462oim4y23mxlh29vx6e1utksvo9tuqsttt0h32o3w2ii87styzrjrfsjbkro1p2ie9v7smi5dyonbqla7/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://4p4icys59003lnxmj07pn1fywo55u1ou1hiaxg7ge1bl5y9liqi2m0ia80hjocsffn70p71m0zwnjjdajj3gewkmy5kn5nwrrwyacyxqpf3288j5elqo237t9qt5r3gm:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 23, "total_num_episodes": 40, "total_success_rate": 0.575, "total_success_rate_lb_0.01": 0.387843728087922, "total_success_rate_ub_0.01": 0.7461782246335723, "total_success_rate_lb_0.025": 0.424182772651577, "total_success_rate_ub_0.025": 0.7285965382595633, "total_success_rate_lb_0.05": 0.44806925208415094, "total_success_rate_ub_0.05": 0.694331735333677, "total_success_rate_lb_0.1": 0.46582129970876485, "total_success_rate_ub_0.1": 0.6801309808728445, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "b9cc57e1cd93555a8ecda2452b7669d638ced38fdd77522d721095e2bc8940d7", "size": 506, "artifact_path": "wandb-client-artifact://scxhztxlxaymzu0ez4dfmr4636cl7qssm2o5hzhfg37ao4kc0eis7sawqffetn7hteg4wd997knzz616mu94teilbswt2874kcf9eof5uu1bdthki3umsbempmwwj0gk/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://48mppkfwo5c8ehmeud7wveosxqutr84q534pw001jlo33oiogqcx0nw8486awvc3ulp14ik3pdcivnebswfc1a5zccvguaajxuio2s29soas376kuq2gyptn2b0f40me:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 278}, "eval/total_success_rate": 0.574999988079071, "eval/total_success_rate_lb_0.01": 0.3878437280654907, "eval/total_success_rate_ub_0.01": 0.7461782097816467, "eval/total_success_rate_lb_0.025": 0.4241827726364136, "eval/total_success_rate_ub_0.025": 0.7285965085029602, "eval/total_success_rate_lb_0.05": 0.4480692446231842, "eval/total_success_rate_ub_0.05": 0.694331705570221, "eval/total_success_rate_lb_0.1": 0.4658212959766388, "eval/total_success_rate_ub_0.1": 0.6801309585571289}
\ No newline at end of file
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/logs/debug-internal.log b/6necfy24/wandb/run-20250101_105741-6necfy24/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..dbbd5a333a459ad34cba7a67ff89c22b4f3064a7
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/logs/debug-internal.log
@@ -0,0 +1,779 @@
+2025-01-01 10:57:41,755 INFO    StreamThr :1788056 [internal.py:wandb_internal():85] W&B internal server running at pid: 1788056, started at: 2025-01-01 10:57:41.755427
+2025-01-01 10:57:41,756 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status
+2025-01-01 10:57:41,768 INFO    WriterThread:1788056 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/run-6necfy24.wandb
+2025-01-01 10:57:41,769 DEBUG   SenderThread:1788056 [sender.py:send():379] send: header
+2025-01-01 10:57:41,778 DEBUG   SenderThread:1788056 [sender.py:send():379] send: run
+2025-01-01 10:57:41,983 INFO    SenderThread:1788056 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files
+2025-01-01 10:57:41,983 INFO    SenderThread:1788056 [sender.py:_start_run_threads():1188] run started: 6necfy24 with start time 1735747061.75616
+2025-01-01 10:57:41,999 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: check_version
+2025-01-01 10:57:41,999 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: check_version
+2025-01-01 10:57:42,043 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: run_start
+2025-01-01 10:57:42,048 DEBUG   HandlerThread:1788056 [system_info.py:__init__():26] System info init
+2025-01-01 10:57:42,048 DEBUG   HandlerThread:1788056 [system_info.py:__init__():41] System info init done
+2025-01-01 10:57:42,048 INFO    HandlerThread:1788056 [system_monitor.py:start():194] Starting system monitor
+2025-01-01 10:57:42,048 INFO    SystemMonitor:1788056 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-01 10:57:42,049 INFO    HandlerThread:1788056 [system_monitor.py:probe():214] Collecting system info
+2025-01-01 10:57:42,049 INFO    SystemMonitor:1788056 [interfaces.py:start():188] Started cpu monitoring
+2025-01-01 10:57:42,050 INFO    SystemMonitor:1788056 [interfaces.py:start():188] Started disk monitoring
+2025-01-01 10:57:42,050 INFO    SystemMonitor:1788056 [interfaces.py:start():188] Started gpu monitoring
+2025-01-01 10:57:42,051 INFO    SystemMonitor:1788056 [interfaces.py:start():188] Started memory monitoring
+2025-01-01 10:57:42,051 INFO    SystemMonitor:1788056 [interfaces.py:start():188] Started network monitoring
+2025-01-01 10:57:42,060 DEBUG   HandlerThread:1788056 [system_info.py:probe():152] Probing system
+2025-01-01 10:57:42,062 DEBUG   HandlerThread:1788056 [system_info.py:_probe_git():137] Probing git
+2025-01-01 10:57:42,066 DEBUG   HandlerThread:1788056 [system_info.py:_probe_git():145] Probing git done
+2025-01-01 10:57:42,066 DEBUG   HandlerThread:1788056 [system_info.py:probe():200] Probing system done
+2025-01-01 10:57:42,066 DEBUG   HandlerThread:1788056 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T15:57:42.060972', 'startedAt': '2025-01-01T15:57:41.743321', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.4958749999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 4500.008, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 657.4940643310547}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-01 10:57:42,066 INFO    HandlerThread:1788056 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-01 10:57:42,066 INFO    HandlerThread:1788056 [system_monitor.py:probe():227] Publishing system info
+2025-01-01 10:57:42,067 DEBUG   HandlerThread:1788056 [system_info.py:_save_code():44] Saving code
+2025-01-01 10:57:42,068 DEBUG   HandlerThread:1788056 [system_info.py:_save_code():67] Saving code done
+2025-01-01 10:57:42,068 DEBUG   HandlerThread:1788056 [system_info.py:_save_patches():84] Saving git patches
+2025-01-01 10:57:42,130 DEBUG   HandlerThread:1788056 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-01 10:57:42,131 INFO    HandlerThread:1788056 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-01 10:57:42,134 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 10:57:42,134 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-01 10:57:42,134 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-01 10:57:42,134 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-01 10:57:42,374 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-01 10:57:42,374 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: python_packages
+2025-01-01 10:57:42,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:57:42,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:57:42,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:42,379 INFO    HandlerThread:1788056 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/tb"
+  save: true
+}
+control {
+  relay_id: "e70ba2374c284bda9de162ecdf68005d"
+}
+_info {
+  stream_id: "6necfy24"
+}
+
+2025-01-01 10:57:42,379 DEBUG   HandlerThread:1788056 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-01 10:57:42,390 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/j1t6nnty-code/FISH/eval_robot.py
+2025-01-01 10:57:42,392 INFO    wandb-upload_0:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/kmz6eb0n-wandb-metadata.json
+2025-01-01 10:57:42,400 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/vzu4x605-diff.patch
+2025-01-01 10:57:42,506 DEBUG   SenderThread:1788056 [sender.py:send():379] send: telemetry
+2025-01-01 10:57:42,506 DEBUG   SenderThread:1788056 [sender.py:send():379] send: tbrecord
+2025-01-01 10:57:42,506 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 10:57:42,506 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0 with policy live
+2025-01-01 10:57:42,689 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/p0ywqfw0-events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:57:42,985 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/code/FISH/eval_robot.py
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/requirements.txt
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-metadata.json
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/diff.patch
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/code/FISH
+2025-01-01 10:57:42,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/code
+2025-01-01 10:57:43,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:44,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:44,984 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:45,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:46,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:47,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:57:47,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:48,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:49,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:49,447 DEBUG   SenderThread:1788056 [sender.py:send():379] send: config
+2025-01-01 10:57:50,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:50,985 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:51,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:52,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:57:52,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:52,986 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:53,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:54,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:54,987 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:55,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:56,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:56,987 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:57,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:57:57,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:57:57,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:57,464 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:57:58,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:57:58,988 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:57:59,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:00,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:01,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:02,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:03,253 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:03,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:04,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:04,990 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:05,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:06,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:06,990 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:07,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:08,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:08,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:08,991 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:09,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:10,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:11,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:12,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:58:12,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:58:12,417 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:13,387 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:13,445 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:13,992 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml
+2025-01-01 10:58:14,388 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:15,388 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:16,388 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:17,388 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:18,388 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:19,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:19,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:20,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:21,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:22,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:23,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:24,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:24,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:25,389 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:26,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:27,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:58:27,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:58:27,417 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:28,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:29,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:29,464 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:30,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:31,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:32,391 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:33,392 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:33,931 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 10:58:33,931 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4 with policy now
+2025-01-01 10:58:33,931 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 10:58:33,932 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 10:58:33,934 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 10:58:33,935 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 10:58:33,997 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4
+2025-01-01 10:58:33,998 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 10:58:33,998 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval
+2025-01-01 10:58:33,998 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos
+2025-01-01 10:58:33,998 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media
+2025-01-01 10:58:34,188 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/eyx99nu8-media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4
+2025-01-01 10:58:34,392 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:34,449 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/0abzwge7-media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4
+2025-01-01 10:58:34,942 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:34,998 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:35,391 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:36,393 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:37,394 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:38,393 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:39,394 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:40,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:40,493 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:41,000 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:41,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:42,051 DEBUG   SystemMonitor:1788056 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-01 10:58:42,052 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 10:58:42,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:58:42,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:58:42,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:43,000 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:43,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:44,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:45,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:46,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:46,491 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:47,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:48,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:49,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:50,003 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:58:50,237 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/djkmibn0-events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:58:50,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:51,003 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:51,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:51,817 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:52,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:53,004 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:53,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:54,396 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:55,005 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:58:55,397 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:56,397 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:57,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:58:57,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:58:57,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:57,527 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:58:58,398 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:58:59,398 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:00,398 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:01,398 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:02,398 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:03,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:03,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:04,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:05,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:06,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:07,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:08,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:08,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:09,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:10,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:11,402 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:12,053 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 10:59:12,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:59:12,375 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:59:12,417 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:13,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:13,440 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:14,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:15,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:16,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:17,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:18,403 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:19,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:19,404 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:20,404 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:21,408 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:21,465 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 10:59:21,465 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 10:59:21,465 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4 with policy now
+2025-01-01 10:59:21,473 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 10:59:21,473 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 10:59:21,475 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 10:59:21,718 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/rhw6z3fz-media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4
+2025-01-01 10:59:22,013 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 10:59:22,013 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4
+2025-01-01 10:59:22,013 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval
+2025-01-01 10:59:22,301 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/thheyvvn-media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4
+2025-01-01 10:59:22,409 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:23,013 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:59:23,405 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:24,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:24,411 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:25,415 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:26,417 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:27,015 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:59:27,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:59:27,378 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:59:27,425 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:28,410 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:29,015 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:59:29,410 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:29,982 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:30,412 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:31,415 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:32,410 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:33,410 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:34,410 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:35,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:35,411 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:36,412 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:37,411 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:38,411 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:39,411 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:40,412 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:41,380 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:41,412 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:42,053 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 10:59:42,377 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:59:42,377 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:59:42,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:43,413 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:44,413 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:45,413 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:46,414 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:46,469 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:47,414 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:48,414 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:49,415 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:50,416 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:51,415 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:52,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:52,414 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:52,812 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 10:59:52,812 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 10:59:52,812 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 10:59:53,024 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:59:53,024 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 10:59:53,267 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/2klaftrg-events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 10:59:53,415 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:54,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:55,025 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:59:55,419 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:56,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:57,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 10:59:57,376 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 10:59:57,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:57,481 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 10:59:58,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 10:59:59,026 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 10:59:59,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:00,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:01,027 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:01,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:02,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:03,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:03,420 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:04,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:05,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:06,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:07,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:08,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:09,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:09,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:10,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:11,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:12,053 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:00:12,376 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:00:12,377 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:00:12,421 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:13,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:14,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:14,459 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:15,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:16,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:17,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:18,422 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:19,423 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:19,917 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:20,423 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:21,034 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:21,423 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:22,423 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:23,437 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:24,435 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:24,745 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 11:00:24,746 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4 with policy now
+2025-01-01 11:00:24,750 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:00:24,750 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:00:24,750 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:00:24,750 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:00:24,991 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/yj2aawlf-media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4
+2025-01-01 11:00:25,035 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:25,036 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:00:25,036 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4
+2025-01-01 11:00:25,036 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval
+2025-01-01 11:00:25,315 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/gq70ldhr-media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4
+2025-01-01 11:00:25,436 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:25,769 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:26,436 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:27,036 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:27,379 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:00:27,379 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:00:27,438 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:28,437 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:29,440 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:30,438 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:30,825 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:31,038 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:31,441 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:32,441 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:33,441 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:34,441 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:35,039 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:35,440 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:36,213 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:36,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:37,040 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:37,447 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:38,447 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:38,647 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:00:38,647 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:00:38,647 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:00:39,040 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:00:39,040 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:00:39,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:40,453 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:41,041 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:41,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:41,613 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:42,054 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:00:42,397 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:00:42,398 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:00:42,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:43,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:44,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:45,042 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:00:45,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:46,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:46,850 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:47,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:48,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:49,449 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:50,450 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:51,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:52,385 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:52,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:53,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:54,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:55,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:56,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:57,378 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:00:57,378 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:00:57,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:57,503 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:00:58,451 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:00:59,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:00,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:01,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:02,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:03,385 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:03,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:04,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:05,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:06,452 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:07,453 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:08,385 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:08,453 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:09,453 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:10,453 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:11,457 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:11,984 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 11:01:11,985 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4 with policy now
+2025-01-01 11:01:11,988 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:01:11,989 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:01:11,989 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:01:11,990 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:01:12,051 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:01:12,051 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4
+2025-01-01 11:01:12,052 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval
+2025-01-01 11:01:12,054 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:01:12,241 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/9u3y1704-media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4
+2025-01-01 11:01:12,384 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:01:12,384 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:01:12,459 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:12,500 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/3nl4yisw-media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4
+2025-01-01 11:01:13,052 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:13,464 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:13,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:14,475 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:15,480 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:16,469 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:17,479 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:18,468 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:19,054 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:19,375 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:19,463 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:20,466 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:21,054 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:21,464 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:22,473 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:23,468 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:24,391 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:24,466 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:25,467 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:26,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:27,386 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:01:27,386 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:01:27,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:28,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:29,057 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:01:29,293 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/mlsq6ykk-events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:01:29,479 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:29,708 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:01:29,709 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:01:29,709 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:29,709 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:01:30,057 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:01:30,479 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:31,058 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:31,473 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:32,474 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:33,058 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:33,469 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:34,469 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:34,929 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:35,059 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:35,469 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:36,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:37,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:38,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:39,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:40,390 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:40,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:41,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:42,055 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:01:42,381 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:01:42,382 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:01:42,470 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:43,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:44,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:45,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:45,525 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:46,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:47,471 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:48,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:49,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:50,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:51,391 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:51,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:52,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:53,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:54,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:55,066 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:01:55,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:56,472 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:56,910 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:01:57,382 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:01:57,382 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:01:57,473 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:58,473 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:01:59,473 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:00,477 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:01,068 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:01,203 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 11:02:01,203 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6071852b701bfb891669.mp4 with policy now
+2025-01-01 11:02:01,205 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:02:01,206 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:02:01,206 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:01,206 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:01,458 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/mby126yl-media/videos/eval/4_eval_4_6071852b701bfb891669.mp4
+2025-01-01 11:02:01,474 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:02,069 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:02,069 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/4_eval_4_6071852b701bfb891669.mp4
+2025-01-01 11:02:02,069 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval
+2025-01-01 11:02:02,238 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:02,369 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/d2f2eaxf-media/videos/eval/4_eval_4_6071852b701bfb891669.mp4
+2025-01-01 11:02:02,486 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:03,069 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:03,483 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:04,484 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:05,490 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:06,485 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:07,071 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:07,395 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:07,494 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:08,494 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:09,497 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:10,499 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:11,072 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:11,498 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:12,055 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:02:12,397 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:02:12,398 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:02:12,496 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:12,499 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:13,496 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:14,498 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:15,499 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:16,503 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:17,513 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:17,654 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-01 11:02:17,654 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: log_artifact
+2025-01-01 11:02:17,832 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:02:18,074 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:02:18,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json
+2025-01-01 11:02:18,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media
+2025-01-01 11:02:18,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table
+2025-01-01 11:02:18,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval
+2025-01-01 11:02:18,124 INFO    wandb-upload_1:1788056 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd7jhhpw5
+2025-01-01 11:02:18,236 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-01 11:02:18,244 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:02:18,286 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/qpqsfr8z-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json
+2025-01-01 11:02:18,497 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:18,598 INFO    SenderThread:1788056 [sender.py:send_request_log_artifact():1525] logged artifact run-6necfy24-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY3MDEzNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTIzOTU4MQ==', 'latestArtifact': None}}
+2025-01-01 11:02:18,598 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 11:02:18,598 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now
+2025-01-01 11:02:18,598 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:18,599 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:02:18,599 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,599 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,600 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:02:18,600 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,603 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,604 DEBUG   SenderThread:1788056 [sender.py:send():379] send: telemetry
+2025-01-01 11:02:18,604 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,604 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,604 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,604 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,605 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,605 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,605 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,605 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,606 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,606 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,606 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,606 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,607 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,607 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,607 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,607 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,608 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,608 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,608 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,608 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,609 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:18,609 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:18,609 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: log_artifact
+2025-01-01 11:02:18,846 INFO    wandb-upload_0:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/pc4x4hed-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json
+2025-01-01 11:02:18,959 INFO    wandb-upload_2:1788056 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpktagczqq
+2025-01-01 11:02:19,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:02:19,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:19,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:19,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json
+2025-01-01 11:02:19,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval
+2025-01-01 11:02:19,339 INFO    SenderThread:1788056 [sender.py:send_request_log_artifact():1525] logged artifact run-6necfy24-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY3MDE0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTIzOTU4NA==', 'latestArtifact': None}}
+2025-01-01 11:02:19,340 DEBUG   SenderThread:1788056 [sender.py:send():379] send: files
+2025-01-01 11:02:19,340 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json with policy now
+2025-01-01 11:02:19,340 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:02:19,340 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:19,341 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:19,497 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:19,518 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/712cyue4-media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json
+2025-01-01 11:02:19,774 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/1xs3cf6p-media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json
+2025-01-01 11:02:20,075 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:20,497 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:20,609 DEBUG   SenderThread:1788056 [sender.py:send():379] send: exit
+2025-01-01 11:02:20,609 INFO    SenderThread:1788056 [sender.py:send_exit():586] handling exit code: 0
+2025-01-01 11:02:20,610 INFO    SenderThread:1788056 [sender.py:send_exit():588] handling runtime: 278
+2025-01-01 11:02:20,610 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:20,610 INFO    SenderThread:1788056 [sender.py:send_exit():594] send defer
+2025-01-01 11:02:20,610 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:20,610 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-01 11:02:20,610 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:20,610 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-01 11:02:20,610 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 1
+2025-01-01 11:02:20,611 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:20,611 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-01 11:02:20,611 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:20,611 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-01 11:02:20,611 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 2
+2025-01-01 11:02:20,611 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:20,611 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-01 11:02:20,611 INFO    HandlerThread:1788056 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-01 11:02:20,611 INFO    HandlerThread:1788056 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-01 11:02:20,612 DEBUG   SystemMonitor:1788056 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-01 11:02:20,612 DEBUG   SystemMonitor:1788056 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-01 11:02:20,612 INFO    HandlerThread:1788056 [interfaces.py:finish():200] Joined disk monitor
+2025-01-01 11:02:20,663 INFO    HandlerThread:1788056 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-01 11:02:20,664 INFO    HandlerThread:1788056 [interfaces.py:finish():200] Joined memory monitor
+2025-01-01 11:02:20,664 INFO    HandlerThread:1788056 [interfaces.py:finish():200] Joined network monitor
+2025-01-01 11:02:20,664 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:20,664 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-01 11:02:20,664 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 3
+2025-01-01 11:02:20,664 DEBUG   SenderThread:1788056 [sender.py:send():379] send: stats
+2025-01-01 11:02:20,664 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:20,665 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-01 11:02:20,665 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:20,665 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-01 11:02:20,665 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 4
+2025-01-01 11:02:20,665 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:20,665 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-01 11:02:21,076 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:21,076 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:23,077 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:26,771 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:02:26,771 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:26,771 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-01 11:02:26,772 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:26,772 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-01 11:02:26,772 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 5
+2025-01-01 11:02:26,772 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:02:26,773 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:26,773 DEBUG   SenderThread:1788056 [sender.py:send():379] send: history
+2025-01-01 11:02:26,773 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-01 11:02:26,773 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:02:26,774 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:26,774 DEBUG   SenderThread:1788056 [sender.py:send():379] send: summary
+2025-01-01 11:02:26,774 INFO    SenderThread:1788056 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:02:26,774 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:26,775 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-01 11:02:26,775 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 6
+2025-01-01 11:02:26,775 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:26,775 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-01 11:02:26,775 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:26,775 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-01 11:02:26,778 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:02:26,878 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 7
+2025-01-01 11:02:26,878 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:26,878 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-01 11:02:26,879 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:26,879 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-01 11:02:27,078 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml
+2025-01-01 11:02:27,078 INFO    Thread-12 :1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:27,610 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:02:30,743 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 8
+2025-01-01 11:02:30,743 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:02:30,744 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:30,744 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-01 11:02:30,744 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:30,744 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-01 11:02:30,744 INFO    SenderThread:1788056 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-01 11:02:30,746 INFO    SenderThread:1788056 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-01 11:02:30,748 INFO    SenderThread:1788056 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-01 11:02:30,757 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 9
+2025-01-01 11:02:30,757 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:30,758 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-01 11:02:30,758 DEBUG   SenderThread:1788056 [sender.py:send():379] send: artifact
+2025-01-01 11:02:30,936 INFO    SenderThread:1788056 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}}
+2025-01-01 11:02:30,937 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:30,937 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-01 11:02:30,937 INFO    SenderThread:1788056 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-01 11:02:31,079 INFO    SenderThread:1788056 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:31,079 INFO    SenderThread:1788056 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml config.yaml
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json wandb-summary.json
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0 events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/requirements.txt requirements.txt
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-metadata.json wandb-metadata.json
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log output.log
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/diff.patch diff.patch
+2025-01-01 11:02:31,080 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-01 11:02:31,084 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/4_eval_4_6071852b701bfb891669.mp4 media/videos/eval/4_eval_4_6071852b701bfb891669.mp4
+2025-01-01 11:02:31,084 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4 media/videos/eval/0_eval_0_259d97e9bef5a9deb167.mp4
+2025-01-01 11:02:31,084 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4 media/videos/eval/3_eval_3_1c141374c1432359d5cd.mp4
+2025-01-01 11:02:31,084 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4 media/videos/eval/1_eval_1_09700a0ebedadd7f6657.mp4
+2025-01-01 11:02:31,084 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4 media/videos/eval/2_eval_2_3ac987b25093947b056a.mp4
+2025-01-01 11:02:31,086 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json media/table/eval/total_success_rate_ci_6_b9cc57e1cd93555a8ecd.table.json
+2025-01-01 11:02:31,086 INFO    SenderThread:1788056 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json
+2025-01-01 11:02:31,086 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 10
+2025-01-01 11:02:31,087 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:31,089 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-01 11:02:31,089 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:31,089 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-01 11:02:31,089 INFO    SenderThread:1788056 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-01 11:02:31,263 INFO    wandb-upload_1:1788056 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/config.yaml
+2025-01-01 11:02:31,283 INFO    wandb-upload_0:1788056 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/wandb-summary.json
+2025-01-01 11:02:31,311 INFO    wandb-upload_4:1788056 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/output.log
+2025-01-01 11:02:31,321 INFO    wandb-upload_2:1788056 [upload_job.py:push():130] Uploaded file /tmp/tmpt9_hvm8owandb/831ywcq5-events.out.tfevents.1735747062.leonmkim-ROG-Strix-G15CS-G15CS.1787880.0
+2025-01-01 11:02:31,338 INFO    wandb-upload_3:1788056 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/files/requirements.txt
+2025-01-01 11:02:31,538 INFO    Thread-11 (_thread_body):1788056 [sender.py:transition_state():614] send defer: 11
+2025-01-01 11:02:31,538 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:31,538 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-01 11:02:31,539 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:31,539 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-01 11:02:31,539 INFO    SenderThread:1788056 [file_pusher.py:join():175] waiting for file pusher
+2025-01-01 11:02:31,540 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 12
+2025-01-01 11:02:31,540 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:31,540 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-01 11:02:31,540 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:31,541 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-01 11:02:31,541 INFO    SenderThread:1788056 [file_stream.py:finish():601] file stream finish called
+2025-01-01 11:02:31,601 INFO    SenderThread:1788056 [file_stream.py:finish():605] file stream finish is done
+2025-01-01 11:02:31,602 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 13
+2025-01-01 11:02:31,602 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:31,602 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-01 11:02:31,602 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:31,602 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-01 11:02:31,602 INFO    SenderThread:1788056 [sender.py:transition_state():614] send defer: 14
+2025-01-01 11:02:31,602 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:02:31,602 INFO    HandlerThread:1788056 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-01 11:02:31,603 DEBUG   SenderThread:1788056 [sender.py:send():379] send: final
+2025-01-01 11:02:31,603 DEBUG   SenderThread:1788056 [sender.py:send():379] send: footer
+2025-01-01 11:02:31,603 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:02:31,603 INFO    SenderThread:1788056 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-01 11:02:31,603 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:02:31,604 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: server_info
+2025-01-01 11:02:31,604 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:02:31,604 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:02:31,604 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: server_info
+2025-01-01 11:02:31,606 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-01 11:02:31,606 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-01 11:02:31,607 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:02:31,658 DEBUG   SenderThread:1788056 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:02:31,658 INFO    MainThread:1788056 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-01 11:02:31,659 INFO    MainThread:1788056 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-01 11:02:31,659 INFO    MainThread:1788056 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-01 11:02:31,659 DEBUG   HandlerThread:1788056 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-01 11:02:31,659 INFO    HandlerThread:1788056 [handler.py:finish():882] shutting down handler
+2025-01-01 11:02:32,605 INFO    WriterThread:1788056 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/run-6necfy24.wandb
+2025-01-01 11:02:32,658 INFO    SenderThread:1788056 [sender.py:finish():1615] shutting down sender
+2025-01-01 11:02:32,658 INFO    SenderThread:1788056 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-01 11:02:32,658 INFO    SenderThread:1788056 [file_pusher.py:join():175] waiting for file pusher
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/logs/debug.log b/6necfy24/wandb/run-20250101_105741-6necfy24/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..8fea950f0879fbe189fa3d451e97050b4bfca392
--- /dev/null
+++ b/6necfy24/wandb/run-20250101_105741-6necfy24/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-01 10:57:41,749 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Configure stats pid to 1787880
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/settings
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/logs/debug.log
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/wandb/run-20250101_105741-6necfy24/logs/debug-internal.log
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:init():569] calling init triggers
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24'}
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:init():619] starting backend
+2025-01-01 10:57:41,750 INFO    MainThread:1787880 [wandb_init.py:init():623] setting up manager
+2025-01-01 10:57:41,754 INFO    MainThread:1787880 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-01 10:57:41,756 INFO    MainThread:1787880 [wandb_init.py:init():631] backend started and connected
+2025-01-01 10:57:41,768 INFO    MainThread:1787880 [wandb_init.py:init():720] updated telemetry
+2025-01-01 10:57:41,775 INFO    MainThread:1787880 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-01 10:57:41,998 INFO    MainThread:1787880 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-01 10:57:42,039 INFO    MainThread:1787880 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-01 10:57:42,039 INFO    MainThread:1787880 [wandb_init.py:init():804] starting run threads in backend
+2025-01-01 10:57:42,375 INFO    MainThread:1787880 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-01 10:57:42,376 INFO    MainThread:1787880 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-01 10:57:42,376 INFO    MainThread:1787880 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-01 10:57:42,376 INFO    MainThread:1787880 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-01 10:57:42,378 INFO    MainThread:1787880 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-01 10:57:42,379 INFO    MainThread:1787880 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/6necfy24/tb, True
+2025-01-01 10:57:49,446 INFO    MainThread:1787880 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'}
+2025-01-01 11:02:32,660 WARNING MsgRouterThr:1787880 [router.py:message_loop():77] message_loop has been closed
diff --git a/6necfy24/wandb/run-20250101_105741-6necfy24/run-6necfy24.wandb b/6necfy24/wandb/run-20250101_105741-6necfy24/run-6necfy24.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..b467deec486af4e3ad5db487041f9cce1c1f6997
Binary files /dev/null and b/6necfy24/wandb/run-20250101_105741-6necfy24/run-6necfy24.wandb differ
diff --git a/7krw5tkq/episode_rosbags/episode_0_2025-01-01-11-29-41.bag b/7krw5tkq/episode_rosbags/episode_0_2025-01-01-11-29-41.bag
new file mode 100644
index 0000000000000000000000000000000000000000..d8d487ab6423fe556c79b4120d986a12fe6293c0
--- /dev/null
+++ b/7krw5tkq/episode_rosbags/episode_0_2025-01-01-11-29-41.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51ce53666c0ed0b0e734b221e26b77fd2d41dc39530134b6819e0e05fe8bd72b
+size 1408444529
diff --git a/7krw5tkq/episode_rosbags/episode_1_2025-01-01-11-30-40.bag b/7krw5tkq/episode_rosbags/episode_1_2025-01-01-11-30-40.bag
new file mode 100644
index 0000000000000000000000000000000000000000..2c02efe92773bfbd4a22bd4c604e07232dbbd122
--- /dev/null
+++ b/7krw5tkq/episode_rosbags/episode_1_2025-01-01-11-30-40.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:430fe13d0deb60c351816df8512bbcb610d76f2c944900199619ddf9aa695669
+size 1410932099
diff --git a/7krw5tkq/episode_rosbags/episode_2_2025-01-01-11-31-40.bag b/7krw5tkq/episode_rosbags/episode_2_2025-01-01-11-31-40.bag
new file mode 100644
index 0000000000000000000000000000000000000000..1848c6bef854f84907854621f23891576f3a0252
--- /dev/null
+++ b/7krw5tkq/episode_rosbags/episode_2_2025-01-01-11-31-40.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:224d097484548a4d0c6ef6849a2fcf2b88d06c64652fd45f25e42f1c08d277d5
+size 1412317030
diff --git a/8511ppcj/.hydra/config.yaml b/8511ppcj/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe
--- /dev/null
+++ b/8511ppcj/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1007_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/8511ppcj/.hydra/hydra.yaml b/8511ppcj/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d1d5ea84c2a41170194cd8ee372210e79735361e
--- /dev/null
+++ b/8511ppcj/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/8511ppcj/.hydra/overrides.yaml b/8511ppcj/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/8511ppcj/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/8511ppcj/episode_rosbags/episode_1_2025-01-01-11-47-39.bag b/8511ppcj/episode_rosbags/episode_1_2025-01-01-11-47-39.bag
new file mode 100644
index 0000000000000000000000000000000000000000..5439fbddff8bf9017eb80a218ac4cb6d2e334e26
--- /dev/null
+++ b/8511ppcj/episode_rosbags/episode_1_2025-01-01-11-47-39.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:302a978583b5070f40dcc0bec918601e538a42ffe09d334c1d85bc7207e62a5b
+size 1414342693
diff --git a/8511ppcj/eval_video/0_eval.mp4 b/8511ppcj/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bbb9d1a6efaf406d951ce87da05f218d81156c17
--- /dev/null
+++ b/8511ppcj/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28199d60beae23009631cdd9a69a0fa00b0c653ee8f2e4691945cdb549c545a9
+size 1168899
diff --git a/8511ppcj/eval_video/1_eval.mp4 b/8511ppcj/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..225c5bf4940b78718f29beda963d4fd686e3d10a
--- /dev/null
+++ b/8511ppcj/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c50442085fb847ef468d3bf301ceb7b72a98283f4b818fc8d6b1067a97e0d240
+size 1167418
diff --git a/8511ppcj/eval_video/2_eval.mp4 b/8511ppcj/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..291ac8dadbfdfee93f7706ef295cb7262c76278c
--- /dev/null
+++ b/8511ppcj/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07e03518140cc7a8fdabaa2beeabd66048d33c7ab437336f65a0a6cf308d8652
+size 1156865
diff --git a/8511ppcj/eval_video/3_eval.mp4 b/8511ppcj/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bba4f7168c2979302b597f0666d94b356ff144f6
--- /dev/null
+++ b/8511ppcj/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9555eba9b6b5e518c2119ec64568c721755a768db904eecbacb52d49129eb08
+size 1172122
diff --git a/8511ppcj/eval_video/4_eval.mp4 b/8511ppcj/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..833cdd719ec5cd246b544bda4ab87060fa6f9d42
--- /dev/null
+++ b/8511ppcj/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c14f84a9e39e1d85ee1dac458ee7be6a41e65ba0a9d832b4085e84956c1262
+size 1202648
diff --git a/8511ppcj/tb/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0 b/8511ppcj/tb/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
new file mode 100644
index 0000000000000000000000000000000000000000..ff36f43ea51df6c1ef70e6b0b01a104f3f51e2f5
--- /dev/null
+++ b/8511ppcj/tb/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c580090370368edfab38c8d91964c21210a6a8ca92fb12df432f78b9a6adf2a
+size 6539
diff --git a/8511ppcj/wandb/debug-internal.log b/8511ppcj/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..67493b0f2bbbc13eda616bdd93f410df70f6663f
--- /dev/null
+++ b/8511ppcj/wandb/debug-internal.log
@@ -0,0 +1,803 @@
+2025-01-01 11:46:17,424 INFO    StreamThr :1797655 [internal.py:wandb_internal():85] W&B internal server running at pid: 1797655, started at: 2025-01-01 11:46:17.423856
+2025-01-01 11:46:17,425 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status
+2025-01-01 11:46:17,437 INFO    WriterThread:1797655 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/run-8511ppcj.wandb
+2025-01-01 11:46:17,437 DEBUG   SenderThread:1797655 [sender.py:send():379] send: header
+2025-01-01 11:46:17,443 DEBUG   SenderThread:1797655 [sender.py:send():379] send: run
+2025-01-01 11:46:17,681 INFO    SenderThread:1797655 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files
+2025-01-01 11:46:17,681 INFO    SenderThread:1797655 [sender.py:_start_run_threads():1188] run started: 8511ppcj with start time 1735749977.424992
+2025-01-01 11:46:17,698 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: check_version
+2025-01-01 11:46:17,698 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: check_version
+2025-01-01 11:46:17,772 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: run_start
+2025-01-01 11:46:17,776 DEBUG   HandlerThread:1797655 [system_info.py:__init__():26] System info init
+2025-01-01 11:46:17,776 DEBUG   HandlerThread:1797655 [system_info.py:__init__():41] System info init done
+2025-01-01 11:46:17,776 INFO    HandlerThread:1797655 [system_monitor.py:start():194] Starting system monitor
+2025-01-01 11:46:17,776 INFO    SystemMonitor:1797655 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-01 11:46:17,777 INFO    HandlerThread:1797655 [system_monitor.py:probe():214] Collecting system info
+2025-01-01 11:46:17,777 INFO    SystemMonitor:1797655 [interfaces.py:start():188] Started cpu monitoring
+2025-01-01 11:46:17,777 INFO    SystemMonitor:1797655 [interfaces.py:start():188] Started disk monitoring
+2025-01-01 11:46:17,778 INFO    SystemMonitor:1797655 [interfaces.py:start():188] Started gpu monitoring
+2025-01-01 11:46:17,779 INFO    SystemMonitor:1797655 [interfaces.py:start():188] Started memory monitoring
+2025-01-01 11:46:17,779 INFO    SystemMonitor:1797655 [interfaces.py:start():188] Started network monitoring
+2025-01-01 11:46:17,789 DEBUG   HandlerThread:1797655 [system_info.py:probe():152] Probing system
+2025-01-01 11:46:17,790 DEBUG   HandlerThread:1797655 [system_info.py:_probe_git():137] Probing git
+2025-01-01 11:46:17,795 DEBUG   HandlerThread:1797655 [system_info.py:_probe_git():145] Probing git done
+2025-01-01 11:46:17,795 DEBUG   HandlerThread:1797655 [system_info.py:probe():200] Probing system done
+2025-01-01 11:46:17,795 DEBUG   HandlerThread:1797655 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:46:17.789371', 'startedAt': '2025-01-01T16:46:17.415416', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3751.5314999999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4505.388, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 724.0196418762207}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-01 11:46:17,795 INFO    HandlerThread:1797655 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-01 11:46:17,795 INFO    HandlerThread:1797655 [system_monitor.py:probe():227] Publishing system info
+2025-01-01 11:46:17,795 DEBUG   HandlerThread:1797655 [system_info.py:_save_code():44] Saving code
+2025-01-01 11:46:17,796 DEBUG   HandlerThread:1797655 [system_info.py:_save_code():67] Saving code done
+2025-01-01 11:46:17,796 DEBUG   HandlerThread:1797655 [system_info.py:_save_patches():84] Saving git patches
+2025-01-01 11:46:17,858 DEBUG   HandlerThread:1797655 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-01 11:46:17,858 INFO    HandlerThread:1797655 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-01 11:46:17,862 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:46:17,862 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-01 11:46:17,862 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-01 11:46:17,862 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-01 11:46:18,094 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-01 11:46:18,094 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:46:18,094 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: python_packages
+2025-01-01 11:46:18,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:46:18,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:18,097 INFO    HandlerThread:1797655 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/tb"
+  save: true
+}
+control {
+  relay_id: "19a8f7b8037f4a2fb815168a54265f74"
+}
+_info {
+  stream_id: "8511ppcj"
+}
+
+2025-01-01 11:46:18,097 DEBUG   HandlerThread:1797655 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-01 11:46:18,123 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/7yjhb4lx-wandb-metadata.json
+2025-01-01 11:46:18,157 DEBUG   SenderThread:1797655 [sender.py:send():379] send: telemetry
+2025-01-01 11:46:18,157 DEBUG   SenderThread:1797655 [sender.py:send():379] send: tbrecord
+2025-01-01 11:46:18,158 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/i7bjapkn-diff.patch
+2025-01-01 11:46:18,158 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:46:18,158 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0 with policy live
+2025-01-01 11:46:18,189 INFO    wandb-upload_1:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/lygq3cfw-code/FISH/eval_robot.py
+2025-01-01 11:46:18,342 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/uxgn6aox-events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:46:18,683 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:46:18,683 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code/FISH/eval_robot.py
+2025-01-01 11:46:18,683 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:18,683 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/diff.patch
+2025-01-01 11:46:18,683 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/requirements.txt
+2025-01-01 11:46:18,684 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-metadata.json
+2025-01-01 11:46:18,684 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code
+2025-01-01 11:46:18,684 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code/FISH
+2025-01-01 11:46:19,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:20,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:20,682 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:21,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:22,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:23,057 DEBUG   SenderThread:1797655 [sender.py:send():379] send: config
+2025-01-01 11:46:23,057 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:23,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:24,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:24,684 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:25,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:26,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:26,684 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:27,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:28,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:28,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:28,685 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:29,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:30,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:30,685 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:31,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:32,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:32,686 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:33,094 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:46:33,094 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:46:33,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:33,181 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:34,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:34,687 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:35,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:36,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:36,687 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:37,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:38,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:38,688 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:39,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:39,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:40,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:41,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:42,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:43,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:44,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:44,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:45,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:46,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:47,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:48,094 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:46:48,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:46:48,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:49,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:49,226 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:49,691 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml
+2025-01-01 11:46:50,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:51,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:52,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:53,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:54,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:55,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:55,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:56,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:56,693 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:57,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:58,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:59,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:00,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:00,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:01,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:02,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:03,094 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:47:03,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:47:03,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:04,101 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:04,383 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:47:04,383 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_28199d60beae23009631.mp4 with policy now
+2025-01-01 11:47:04,384 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:47:04,384 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:47:04,385 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:47:04,385 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:47:04,641 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/bsau7ev0-media/videos/eval/0_eval_0_28199d60beae23009631.mp4
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/0_eval_0_28199d60beae23009631.mp4
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval
+2025-01-01 11:47:04,953 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/wu0d40av-media/videos/eval/0_eval_0_28199d60beae23009631.mp4
+2025-01-01 11:47:05,101 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:05,394 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:06,101 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:06,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:07,101 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:08,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:09,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:10,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:10,437 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:10,697 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:11,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:12,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:12,698 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:13,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:14,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:15,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:16,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:16,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:17,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:17,780 DEBUG   SystemMonitor:1797655 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-01 11:47:17,781 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:47:18,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:47:18,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:47:18,141 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:19,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:20,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:21,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:21,216 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:22,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:23,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:24,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:25,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:26,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:27,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:27,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:28,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:29,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:30,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:31,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:32,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:32,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:33,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:47:33,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:47:33,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:34,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:35,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:35,705 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:47:35,944 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/ssnj77ue-events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:47:36,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:36,705 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:37,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:37,449 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:38,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:38,706 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:39,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:40,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:40,707 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:41,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:42,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:43,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:43,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:44,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:45,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:46,107 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:47,107 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:47,781 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:47:48,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:47:48,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:47:48,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:48,245 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:49,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:50,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:51,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:52,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:53,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:54,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:54,109 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:55,109 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:56,109 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:57,109 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:58,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:59,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:59,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:00,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:01,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:02,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:03,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:48:03,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:48:03,141 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:04,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:04,202 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:05,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:06,113 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:07,113 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:07,444 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:48:07,444 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4 with policy now
+2025-01-01 11:48:07,445 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:48:07,445 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:48:07,445 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:48:07,446 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:48:07,705 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/hpl97859-media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4
+2025-01-01 11:48:07,715 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:48:07,715 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4
+2025-01-01 11:48:07,716 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval
+2025-01-01 11:48:07,999 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/27flny6g-media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4
+2025-01-01 11:48:08,113 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:08,716 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:09,113 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:09,716 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:10,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:10,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:11,116 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:12,113 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:12,717 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:13,114 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:14,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:15,119 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:15,409 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:16,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:16,718 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:17,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:17,782 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:48:18,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:48:18,096 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:48:18,141 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:19,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:20,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:21,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:21,157 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:22,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:23,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:23,525 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:48:23,526 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:48:23,527 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:48:23,721 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:48:23,721 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:48:24,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:24,721 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:25,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:26,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:26,483 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:26,722 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:27,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:28,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:28,722 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:29,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:30,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:31,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:32,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:32,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:33,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:48:33,096 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:48:33,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:34,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:35,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:36,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:37,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:37,223 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:38,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:39,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:40,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:41,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:42,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:43,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:43,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:44,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:45,124 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:46,124 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:47,124 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:47,782 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:48:48,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:48:48,096 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:48:48,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:48,186 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:49,125 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:50,125 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:51,125 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:52,125 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:52,730 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:53,125 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:53,424 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:54,128 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:54,659 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:48:54,659 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4 with policy now
+2025-01-01 11:48:54,660 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:48:54,660 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:48:54,660 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:48:54,661 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:48:54,731 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:48:54,731 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:54,731 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4
+2025-01-01 11:48:54,731 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval
+2025-01-01 11:48:54,922 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/2s1mhs27-media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4
+2025-01-01 11:48:55,133 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:55,181 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/e1ielie5-media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4
+2025-01-01 11:48:56,132 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:56,731 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:57,133 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:58,144 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:58,705 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:59,133 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:00,135 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:00,733 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:01,140 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:02,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:02,733 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:03,102 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:49:03,102 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:49:03,145 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:04,134 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:04,171 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:05,133 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:06,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:07,139 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:08,143 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:09,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:10,107 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:10,139 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:11,141 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:12,139 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:13,139 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:14,142 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:15,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:15,143 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:16,145 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:17,144 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:17,782 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:49:18,102 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:49:18,103 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:49:18,149 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:19,144 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:20,145 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:20,176 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:21,149 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:22,149 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:22,372 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:49:22,373 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:49:22,373 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:49:22,740 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:49:22,740 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:49:22,989 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/liwo1zvs-events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:49:23,148 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:24,152 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:24,741 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:25,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:25,277 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:26,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:27,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:28,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:28,742 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:29,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:30,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:30,743 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:30,990 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:31,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:32,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:33,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:49:33,098 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:49:33,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:34,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:35,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:36,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:36,239 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:37,155 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:38,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:39,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:40,152 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:41,152 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:42,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:42,152 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:43,152 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:44,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:45,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:46,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:47,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:47,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:47,783 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:49:48,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:49:48,098 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:49:48,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:49,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:50,154 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:51,154 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:52,154 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:52,212 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:53,173 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:54,156 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:54,281 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:49:54,281 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4 with policy now
+2025-01-01 11:49:54,289 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:49:54,290 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:49:54,290 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:49:54,291 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:49:54,578 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/qhh694eq-media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4
+2025-01-01 11:49:54,751 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:49:54,751 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:54,751 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4
+2025-01-01 11:49:54,752 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval
+2025-01-01 11:49:55,008 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/18v4tl5l-media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4
+2025-01-01 11:49:55,160 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:56,175 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:56,752 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:57,165 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:57,323 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:58,166 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:59,168 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:00,169 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:00,753 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:01,173 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:02,168 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:02,754 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:03,101 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:50:03,102 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:50:03,167 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:03,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:04,166 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:05,175 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:06,177 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:07,178 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:08,172 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:09,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:09,178 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:10,179 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:11,180 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:12,182 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:13,185 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:14,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:14,187 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:15,189 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:16,188 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:17,186 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:17,783 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:50:18,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:50:18,138 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:50:18,189 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:18,439 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:50:18,439 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:50:18,439 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:50:18,759 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:50:18,759 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:50:19,155 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:19,197 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:20,192 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:20,760 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:21,198 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:22,194 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:23,217 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:24,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:24,440 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:24,761 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:25,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:26,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:27,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:28,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:29,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:30,114 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:30,194 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:31,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:32,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:33,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:50:33,099 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:50:33,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:34,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:35,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:35,254 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:36,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:37,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:38,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:38,766 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:39,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:40,196 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:41,118 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:41,196 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:42,196 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:43,196 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:44,196 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:45,199 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:46,197 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:47,118 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:47,197 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:47,784 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:50:48,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:50:48,099 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:50:48,197 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:49,198 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:50,198 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:51,204 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:51,529 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:50:51,529 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4 with policy now
+2025-01-01 11:50:51,535 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:50:51,536 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:50:51,536 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:50:51,537 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:50:51,770 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:50:51,770 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4
+2025-01-01 11:50:51,771 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval
+2025-01-01 11:50:51,790 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/0iu5p2o0-media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4
+2025-01-01 11:50:52,050 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/j32q2l82-media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4
+2025-01-01 11:50:52,202 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:52,556 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:52,771 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:53,206 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:53,771 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:54,214 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:55,206 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:56,213 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:56,772 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:57,203 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:57,605 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:58,225 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:59,216 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:00,215 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:00,773 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:01,204 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:02,209 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:03,116 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:51:03,116 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:51:03,208 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:51:03,209 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:04,211 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:05,222 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:06,222 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:07,222 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:08,214 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:08,499 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:51:08,499 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:08,499 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:51:08,499 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:08,776 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:51:08,776 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:51:08,776 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:08,913 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-01 11:51:08,913 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: log_artifact
+2025-01-01 11:51:09,061 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/pma5ojbp-events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:51:09,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:51:09,215 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:09,425 INFO    wandb-upload_0:1797655 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1b9puqx6
+2025-01-01 11:51:09,551 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-01 11:51:09,559 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval
+2025-01-01 11:51:09,952 INFO    SenderThread:1797655 [sender.py:send_request_log_artifact():1525] logged artifact run-8511ppcj-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcwMzQ5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDQ0Mg==', 'latestArtifact': None}}
+2025-01-01 11:51:09,952 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:51:09,952 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2025-01-01 11:51:09,953 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:51:09,953 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,954 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,954 DEBUG   SenderThread:1797655 [sender.py:send():379] send: telemetry
+2025-01-01 11:51:09,954 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,958 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,958 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,958 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,958 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,959 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,959 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,959 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,959 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,960 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,960 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,960 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,960 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,961 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,961 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,961 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,961 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,961 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,962 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,962 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,962 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,962 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,963 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: log_artifact
+2025-01-01 11:51:10,126 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/7hxr9auz-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-01 11:51:10,167 INFO    wandb-upload_1:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/79iihm2m-media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json
+2025-01-01 11:51:10,215 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:10,332 INFO    wandb-upload_0:1797655 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7k2ccjgz
+2025-01-01 11:51:10,371 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/tzw4ycwg-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-01 11:51:10,713 INFO    SenderThread:1797655 [sender.py:send_request_log_artifact():1525] logged artifact run-8511ppcj-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcwMzUyMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDQ0Nw==', 'latestArtifact': None}}
+2025-01-01 11:51:10,714 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:51:10,714 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json with policy now
+2025-01-01 11:51:10,714 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:51:10,714 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:10,715 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:10,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:51:10,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:10,984 INFO    wandb-upload_1:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/cn7k02se-media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json
+2025-01-01 11:51:11,215 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:11,721 DEBUG   SenderThread:1797655 [sender.py:send():379] send: exit
+2025-01-01 11:51:11,722 INFO    SenderThread:1797655 [sender.py:send_exit():586] handling exit code: 0
+2025-01-01 11:51:11,722 INFO    SenderThread:1797655 [sender.py:send_exit():588] handling runtime: 293
+2025-01-01 11:51:11,722 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:11,722 INFO    SenderThread:1797655 [sender.py:send_exit():594] send defer
+2025-01-01 11:51:11,722 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:11,722 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-01 11:51:11,722 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:11,722 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-01 11:51:11,723 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 1
+2025-01-01 11:51:11,723 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:11,723 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-01 11:51:11,723 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:11,723 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-01 11:51:11,723 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 2
+2025-01-01 11:51:11,723 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:11,723 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-01 11:51:11,723 INFO    HandlerThread:1797655 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-01 11:51:11,723 DEBUG   SystemMonitor:1797655 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-01 11:51:11,723 DEBUG   SystemMonitor:1797655 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-01 11:51:11,724 INFO    HandlerThread:1797655 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-01 11:51:11,724 INFO    HandlerThread:1797655 [interfaces.py:finish():200] Joined disk monitor
+2025-01-01 11:51:11,777 INFO    HandlerThread:1797655 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-01 11:51:11,777 INFO    HandlerThread:1797655 [interfaces.py:finish():200] Joined memory monitor
+2025-01-01 11:51:11,777 INFO    HandlerThread:1797655 [interfaces.py:finish():200] Joined network monitor
+2025-01-01 11:51:11,778 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:11,778 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-01 11:51:11,778 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 3
+2025-01-01 11:51:11,778 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:51:11,778 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:11,778 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-01 11:51:11,778 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:11,778 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-01 11:51:11,778 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 4
+2025-01-01 11:51:11,779 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:11,779 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-01 11:51:11,779 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:51:12,780 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:18,504 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:51:18,504 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:18,505 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:51:18,505 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-01 11:51:18,505 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-01 11:51:18,505 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 5
+2025-01-01 11:51:18,506 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:51:18,506 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:18,506 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-01 11:51:18,506 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:51:18,507 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:18,507 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:18,507 DEBUG   SenderThread:1797655 [sender.py:send():379] send: summary
+2025-01-01 11:51:18,508 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:18,508 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:18,508 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-01 11:51:18,508 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 6
+2025-01-01 11:51:18,508 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:18,508 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-01 11:51:18,509 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:18,509 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-01 11:51:18,512 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:51:18,635 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 7
+2025-01-01 11:51:18,636 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:18,636 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-01 11:51:18,636 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:18,636 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-01 11:51:18,723 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:51:18,782 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:51:18,782 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml
+2025-01-01 11:51:21,623 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 8
+2025-01-01 11:51:21,623 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:51:21,624 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:21,624 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-01 11:51:21,624 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:21,624 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-01 11:51:21,624 INFO    SenderThread:1797655 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-01 11:51:21,626 INFO    SenderThread:1797655 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-01 11:51:21,628 INFO    SenderThread:1797655 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-01 11:51:21,637 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 9
+2025-01-01 11:51:21,637 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:21,637 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-01 11:51:21,637 DEBUG   SenderThread:1797655 [sender.py:send():379] send: artifact
+2025-01-01 11:51:21,724 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:51:21,783 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:21,816 INFO    SenderThread:1797655 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}}
+2025-01-01 11:51:21,816 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:21,816 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-01 11:51:21,816 INFO    SenderThread:1797655 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-01 11:51:22,784 INFO    SenderThread:1797655 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files
+2025-01-01 11:51:22,784 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml config.yaml
+2025-01-01 11:51:22,784 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json wandb-summary.json
+2025-01-01 11:51:22,784 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/requirements.txt requirements.txt
+2025-01-01 11:51:22,785 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-metadata.json wandb-metadata.json
+2025-01-01 11:51:22,785 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0 events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log output.log
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/diff.patch diff.patch
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/0_eval_0_28199d60beae23009631.mp4 media/videos/eval/0_eval_0_28199d60beae23009631.mp4
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4 media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4 media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4
+2025-01-01 11:51:22,791 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4 media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4
+2025-01-01 11:51:22,791 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4 media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4
+2025-01-01 11:51:22,793 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json
+2025-01-01 11:51:22,794 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-01 11:51:22,794 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 10
+2025-01-01 11:51:22,794 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:51:22,794 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:22,794 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-01 11:51:22,794 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:22,794 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-01 11:51:22,794 INFO    SenderThread:1797655 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-01 11:51:22,968 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:51:22,984 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml
+2025-01-01 11:51:22,995 INFO    wandb-upload_1:1797655 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/requirements.txt
+2025-01-01 11:51:22,997 INFO    wandb-upload_4:1797655 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:23,093 INFO    wandb-upload_3:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/p1bqef6k-events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:51:23,294 INFO    Thread-11 (_thread_body):1797655 [sender.py:transition_state():614] send defer: 11
+2025-01-01 11:51:23,294 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:23,294 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-01 11:51:23,294 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:23,294 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-01 11:51:23,294 INFO    SenderThread:1797655 [file_pusher.py:join():175] waiting for file pusher
+2025-01-01 11:51:23,295 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 12
+2025-01-01 11:51:23,296 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:23,296 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-01 11:51:23,296 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:23,296 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-01 11:51:23,296 INFO    SenderThread:1797655 [file_stream.py:finish():601] file stream finish called
+2025-01-01 11:51:23,533 INFO    SenderThread:1797655 [file_stream.py:finish():605] file stream finish is done
+2025-01-01 11:51:23,533 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 13
+2025-01-01 11:51:23,534 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:23,534 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-01 11:51:23,534 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:23,534 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-01 11:51:23,534 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 14
+2025-01-01 11:51:23,534 DEBUG   SenderThread:1797655 [sender.py:send():379] send: final
+2025-01-01 11:51:23,534 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:23,534 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-01 11:51:23,534 DEBUG   SenderThread:1797655 [sender.py:send():379] send: footer
+2025-01-01 11:51:23,535 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:23,535 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-01 11:51:23,535 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:51:23,535 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: server_info
+2025-01-01 11:51:23,535 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:51:23,536 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:51:23,536 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-01 11:51:23,536 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: server_info
+2025-01-01 11:51:23,536 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-01 11:51:23,538 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:23,586 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:51:23,586 INFO    MainThread:1797655 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-01 11:51:23,586 INFO    MainThread:1797655 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-01 11:51:23,587 INFO    MainThread:1797655 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-01 11:51:23,587 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-01 11:51:23,587 INFO    HandlerThread:1797655 [handler.py:finish():882] shutting down handler
+2025-01-01 11:51:24,536 INFO    WriterThread:1797655 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/run-8511ppcj.wandb
+2025-01-01 11:51:24,586 INFO    SenderThread:1797655 [sender.py:finish():1615] shutting down sender
+2025-01-01 11:51:24,586 INFO    SenderThread:1797655 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-01 11:51:24,586 INFO    SenderThread:1797655 [file_pusher.py:join():175] waiting for file pusher
diff --git a/8511ppcj/wandb/debug.log b/8511ppcj/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..ffe5e10d7d63b07805f5db75bc08fa5b9db95a44
--- /dev/null
+++ b/8511ppcj/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Configure stats pid to 1797490
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/settings
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/logs/debug.log
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/logs/debug-internal.log
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_init.py:init():569] calling init triggers
+2025-01-01 11:46:17,419 INFO    MainThread:1797490 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj'}
+2025-01-01 11:46:17,419 INFO    MainThread:1797490 [wandb_init.py:init():619] starting backend
+2025-01-01 11:46:17,419 INFO    MainThread:1797490 [wandb_init.py:init():623] setting up manager
+2025-01-01 11:46:17,423 INFO    MainThread:1797490 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-01 11:46:17,424 INFO    MainThread:1797490 [wandb_init.py:init():631] backend started and connected
+2025-01-01 11:46:17,436 INFO    MainThread:1797490 [wandb_init.py:init():720] updated telemetry
+2025-01-01 11:46:17,441 INFO    MainThread:1797490 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-01 11:46:17,697 INFO    MainThread:1797490 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-01 11:46:17,767 INFO    MainThread:1797490 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-01 11:46:17,767 INFO    MainThread:1797490 [wandb_init.py:init():804] starting run threads in backend
+2025-01-01 11:46:18,095 INFO    MainThread:1797490 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-01 11:46:18,095 INFO    MainThread:1797490 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-01 11:46:18,095 INFO    MainThread:1797490 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-01 11:46:18,095 INFO    MainThread:1797490 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-01 11:46:18,096 INFO    MainThread:1797490 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-01 11:46:18,097 INFO    MainThread:1797490 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/tb, True
+2025-01-01 11:46:23,056 INFO    MainThread:1797490 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'}
+2025-01-01 11:51:24,587 WARNING MsgRouterThr:1797490 [router.py:message_loop():77] message_loop has been closed
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code/FISH/eval_robot.py b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..acabef87152da9642cb98151c64fc4255609b548
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1007_0'
+wandb_notes:
+  desc: null
+  value: 1007_0_restarted_2
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1735749977
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: lady
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/diff.patch b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..eff8de3c864ce1e6f5e6f3000cbf990769adf78e
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/diff.patch
@@ -0,0 +1,105 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 0f6a9ec..b9e3dfe 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
++# wandb_run_id: '1002_0' 
++
++# all books, 6/20 demos per book
++# RGBD+mask+act history
+ wandb_run_id: '1007_0' 
+ 
++# RGBD+mask+contact(all ftrs)+act history
++# wandb_run_id: '1006_0'
++
++# RGBD+mask+contact(contact+DTC)+act history
++# wandb_run_id: '1745_0'
+ 
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 21e335c..71a5d1c 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -155,7 +155,12 @@ run_id_list = [
+     # '1022_0',
+     # '1023_0',
+     # '1007_0',
+-    '1007_0'
++    # '1007_0'
++]
++
++run_id_list = [
++    # '1006_0',
++    '1745_0'
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py
+index 4179186..50493fe 100644
+--- a/FISH/eval_robot.py
++++ b/FISH/eval_robot.py
+@@ -478,8 +478,8 @@ class Workspace:
+ 				"summary_metrics.episode": {"$gte": 5},
+ 				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+ 				"state": "finished",
+-				"config.grasped_obj_name": grasped_obj_name,
+-				"config.left_book_slot": left_book_slot,
++				# "config.grasped_obj_name": grasped_obj_name,
++				# "config.left_book_slot": left_book_slot,
+ 			}
+ 
+ 			api = wandb.Api()
+@@ -490,7 +490,8 @@ class Workspace:
+ 			if len(filtered_runs) > 0:
+ 				for filtered_run in filtered_runs:
+ 					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+-					total_num_episodes += filtered_run.summary_metrics['episode']
++					# total_num_episodes += filtered_run.summary_metrics['episode']
++					total_num_episodes += filtered_run.config['num_eval']
+ 					list_of_historical_run_ids.append(filtered_run.id)
+ 
+ 			wandb.summary['total_num_successes'] = total_num_successes
+@@ -513,7 +514,6 @@ class Workspace:
+ 				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+ 				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+ 
+-
+ 			total_table_data = [total_table_data]
+ 			wandb.log({
+ 				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py
+index 3302c31..1946eca 100644
+--- a/gym-envs/gym_envs/envs/franka_env.py
++++ b/gym-envs/gym_envs/envs/franka_env.py
+@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env):
+ 
+ 	def mask_obs_callback(self, msg):
+ 		msg = self.possibly_resize_mask_obs(msg)
+-		rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}")
++		# rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}")
+ 		self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg)  
+ 
+ 	def contact_obs_callback(self, msg):
+diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py
+index e3031af..6f97e26 100644
+--- a/gym-envs/gym_envs/envs/hardware/franka.py
++++ b/gym-envs/gym_envs/envs/hardware/franka.py
+@@ -330,11 +330,11 @@ class Franka:
+ 		# self.yaw = yaw
+ 
+ 		self.move_to_target_hz = 10.0
+-		self.move_to_trans_tolerance = 0.01
+-		self.move_to_rot_tolerance = 2.0
++		self.move_to_trans_tolerance = 0.005
++		self.move_to_rot_tolerance = 1.0
+ 		self.move_to_target_max_velocity = 0.8
+ 		self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz)
+-		self.move_to_target_timeout = 15.0
++		self.move_to_target_timeout = 5.0
+ 		self.move_to_target_max_rotation = 25.0
+ 		# only can use one of these at a time
+ 		# assert use_roll + use_pitch + use_yaw <= 1, "Can only use one of roll, pitch, yaw"
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]}
\ No newline at end of file
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..a13da587d5a8a1ad618ba1081fd02a6ffa04fd10
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[33, 60, 0.55, 0.39797851445331156, 0.6944345235435715, 0.42656133325376633, 0.6780519187094306, 0.4461565911877554, 0.6503903641996011, 0.4621801376418414, 0.6364702135051538]]}
\ No newline at end of file
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/0_eval_0_28199d60beae23009631.mp4 b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/0_eval_0_28199d60beae23009631.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bbb9d1a6efaf406d951ce87da05f218d81156c17
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/0_eval_0_28199d60beae23009631.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28199d60beae23009631cdd9a69a0fa00b0c653ee8f2e4691945cdb549c545a9
+size 1168899
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4 b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..225c5bf4940b78718f29beda963d4fd686e3d10a
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c50442085fb847ef468d3bf301ceb7b72a98283f4b818fc8d6b1067a97e0d240
+size 1167418
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4 b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..291ac8dadbfdfee93f7706ef295cb7262c76278c
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07e03518140cc7a8fdabaa2beeabd66048d33c7ab437336f65a0a6cf308d8652
+size 1156865
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4 b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bba4f7168c2979302b597f0666d94b356ff144f6
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9555eba9b6b5e518c2119ec64568c721755a768db904eecbacb52d49129eb08
+size 1172122
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4 b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..833cdd719ec5cd246b544bda4ab87060fa6f9d42
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c14f84a9e39e1d85ee1dac458ee7be6a41e65ba0a9d832b4085e84956c1262
+size 1202648
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..cfe356404d21756db523dbcea380190e97c2c9ed
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
@@ -0,0 +1,268 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1735749983.057114]: resetting environment
+[INFO] [1735749983.060501]: cleared current plan
+[INFO] [1735749983.061467]: moving to home
+[INFO] [1735749984.064224]: reached home
+[INFO] [1735749984.065622]: reset action history
+[INFO] [1735749986.775126]: environment reset
+Starting episode 0
+[INFO] [1735749986.777098]: resetting environment
+[INFO] [1735749986.781155]: cleared current plan
+[INFO] [1735749986.781979]: moving to home
+[INFO] [1735749987.784495]: reached home
+[INFO] [1735749987.785722]: reset action history
+[INFO] [1735749990.493044]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1735749992.688633]: resetting environment
+[INFO] [1735749992.691684]: cleared current plan
+[INFO] [1735749992.692658]: moving to home
+[INFO] [1735749993.695272]: reached home
+[INFO] [1735749993.696656]: reset action history
+[INFO] [1735749996.404867]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/episode_rosbags/episode_0_2025-01-01-11-46-36.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1797490.log
+started roslaunch server http://158.130.50.37:40187/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [1797766]
+process[print_text-2]: started with pid [1797790]
+process[rosbag_record-3]: started with pid [1797791]
+started bagging!
+For topic gripper_width: timestamp difference is 103760719 for nearest: 1735750015445279598 - target: 1735750015341518879 at idx 246
+[INFO] [1735750023.400421]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1735750024.154795]: Stored episode 1.
+[INFO] [1735750024.155745]: Saving video...
+[INFO] [1735750024.368426]: Video saved!
+Starting episode 1
+[INFO] [1735750024.385119]: resetting environment
+[INFO] [1735750024.393388]: cleared current plan
+[INFO] [1735750024.393668]: moving to home
+[INFO] [1735750028.396524]: reached home
+[INFO] [1735750028.397031]: reset action history
+[INFO] [1735750031.106308]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735750055.398269]: resetting environment
+[INFO] [1735750055.400431]: cleared current plan
+[INFO] [1735750055.400581]: moving to home
+[INFO] [1735750056.401300]: reached home
+[INFO] [1735750056.401730]: reset action history
+[INFO] [1735750059.108628]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/episode_rosbags/episode_1_2025-01-01-11-47-39.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1797490.log
+started roslaunch server http://158.130.50.37:33141/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [1797902]
+process[print_text-5]: started with pid [1797903]
+process[rosbag_record-6]: started with pid [1797927]
+started bagging!
+[INFO] [1735750086.067432]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1735750086.845509]: Stored episode 2.
+[INFO] [1735750086.845750]: Saving video...
+[INFO] [1735750087.394113]: Video saved!
+Starting episode 2
+[INFO] [1735750087.451721]: resetting environment
+[INFO] [1735750087.470703]: cleared current plan
+[INFO] [1735750087.471356]: moving to home
+[WARN] [1735750087.425144]: Plan exhausted
+[WARN] [1735750087.464971]: Plan exhausted
+[INFO] [1735750091.574971]: reached home
+[INFO] [1735750091.578140]: reset action history
+[INFO] [1735750094.367972]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735750102.765189]: resetting environment
+[INFO] [1735750102.772220]: cleared current plan
+[INFO] [1735750102.772395]: moving to home
+[INFO] [1735750103.773269]: reached home
+[INFO] [1735750103.774970]: reset action history
+[INFO] [1735750106.483072]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/episode_rosbags/episode_2_2025-01-01-11-48-26.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1797490.log
+started roslaunch server http://158.130.50.37:41799/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [1798039]
+process[print_text-8]: started with pid [1798040]
+process[rosbag_record-9]: started with pid [1798041]
+started bagging!
+For topic gripper_width: timestamp difference is 103145764 for nearest: 1735750130645282195 - target: 1735750130542136431 at idx 249
+[INFO] [1735750133.423526]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1735750134.219486]: Stored episode 3.
+[INFO] [1735750134.224429]: Saving video...
+[INFO] [1735750134.624085]: Video saved!
+Starting episode 3
+[INFO] [1735750134.664512]: resetting environment
+[INFO] [1735750134.688082]: cleared current plan
+[INFO] [1735750134.696096]: moving to home
+[INFO] [1735750138.704833]: reached home
+[INFO] [1735750138.715111]: reset action history
+[INFO] [1735750141.437237]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735750162.211810]: resetting environment
+[INFO] [1735750162.227468]: cleared current plan
+[INFO] [1735750162.229321]: moving to home
+[INFO] [1735750163.233336]: reached home
+[INFO] [1735750163.237625]: reset action history
+[INFO] [1735750166.017539]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/episode_rosbags/episode_3_2025-01-01-11-49-26.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1797490.log
+started roslaunch server http://158.130.50.37:38145/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [1798214]
+process[print_text-11]: started with pid [1798215]
+process[rosbag_record-12]: started with pid [1798239]
+started bagging!
+For topic gripper_width: timestamp difference is 101904515 for nearest: 1735750168946314219 - target: 1735750168844409704 at idx 43
+[INFO] [1735750193.126772]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1735750193.885273]: Stored episode 4.
+[INFO] [1735750193.894614]: Saving video...
+[INFO] [1735750194.226146]: Video saved!
+Starting episode 4
+[INFO] [1735750194.292270]: resetting environment
+[INFO] [1735750194.319056]: cleared current plan
+[INFO] [1735750194.321730]: moving to home
+[INFO] [1735750198.334198]: reached home
+[INFO] [1735750198.339665]: reset action history
+[INFO] [1735750201.126730]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735750218.124275]: resetting environment
+[INFO] [1735750218.145588]: cleared current plan
+[INFO] [1735750218.148251]: moving to home
+[INFO] [1735750219.155201]: reached home
+[INFO] [1735750219.155982]: reset action history
+[INFO] [1735750221.936575]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/episode_rosbags/episode_4_2025-01-01-11-50-21.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1797490.log
+started roslaunch server http://158.130.50.37:33423/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [1798373]
+process[print_text-14]: started with pid [1798374]
+process[rosbag_record-15]: started with pid [1798375]
+started bagging!
+For topic gripper_width: timestamp difference is -111389739 for nearest: 1735750236178613561 - target: 1735750236290003300 at idx 212
+[INFO] [1735750250.155717]: Storing episode...
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit
+[print_text-14] killing on exit
+[INFO] [1735750250.945472]: Stored episode 5.
+[INFO] [1735750250.956426]: Saving video...
+[INFO] [1735750251.462858]: Video saved!
+[INFO] [1735750251.536410]: resetting environment
+[INFO] [1735750251.555546]: cleared current plan
+[INFO] [1735750251.555973]: moving to home
+[WARN] [1735750251.504978]: Plan exhausted
+[WARN] [1735750251.544996]: Plan exhausted
+[INFO] [1735750255.562120]: reached home
+[INFO] [1735750255.567268]: reset action history
+[INFO] [1735750258.528566]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/requirements.txt b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-metadata.json b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..ac29d89746f28b5300f84cfb3460451eaa8491da
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-01T16:46:17.789371",
+    "startedAt": "2025-01-01T16:46:17.415416",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3751.5314999999996,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 4505.388,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 724.0196418762207
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..8061d7a8a23f01e33eea07db9b7104af2df3023e
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "28199d60beae23009631cdd9a69a0fa00b0c653ee8f2e4691945cdb549c545a9", "size": 1168899, "path": "media/videos/eval/0_eval_0_28199d60beae23009631.mp4"}, "global_step": 665, "_timestamp": 1735750269.469801, "_runtime": 292.0448088645935, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "c50442085fb847ef468d3bf301ceb7b72a98283f4b818fc8d6b1067a97e0d240", "size": 1167418, "path": "media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4"}, "eval/num_success": 5.0, "episode": 60.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "07e03518140cc7a8fdabaa2beeabd66048d33c7ab437336f65a0a6cf308d8652", "size": 1156865, "path": "media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c9555eba9b6b5e518c2119ec64568c721755a768db904eecbacb52d49129eb08", "size": 1172122, "path": "media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "77c14f84a9e39e1d85ee1dac458ee7be6a41e65ba0a9d832b4085e84956c1262", "size": 1202648, "path": "media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://6rp7nialc9xie26zx5b8u1dvt2p0o7wzegptx6me38iig786wtf9r3h9bmz37go04jbutp0pn6x1h98ntlogsudebjv2o0ecanklca81xjrmic6ed560yb9bwf9ibkik/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://eovmpnwxau81d600bylz35gjmqiormxg6k8iv0bi8ozpd4xujddfz79yg8csorb898tk88nya5dtsr4rvsjlkcelh27i3fd5lujtdg1w3oo24u2yu7up5tcjrmd9g2tq:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 33, "total_num_episodes": 60, "total_success_rate": 0.55, "total_success_rate_lb_0.01": 0.39797851445331156, "total_success_rate_ub_0.01": 0.6944345235435715, "total_success_rate_lb_0.025": 0.42656133325376633, "total_success_rate_ub_0.025": 0.6780519187094306, "total_success_rate_lb_0.05": 0.4461565911877554, "total_success_rate_ub_0.05": 0.6503903641996011, "total_success_rate_lb_0.1": 0.4621801376418414, "total_success_rate_ub_0.1": 0.6364702135051538, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "101bd877386d563fcdb4fca163c1433dcecd77c51dda3345582a6e75a030dbb9", "size": 508, "artifact_path": "wandb-client-artifact://jpgqvxjg6o3ngy9i8redwt9kwplj1f938vpemran3gkuengaxwmtcxyw5p6ywkiiwbsz3o84a51uoywkvyrfpnzwhh4vt6r4lgp2f76pcq6ornx3fgjirw09vykrzey8/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vkaitd4vzznaxf44ytdldqzjvw95batx0idvak0tlaazrpvqxdozz30fqzokb9gcckiyuwhnlal057918zceficerwslzk4cnjdyunifj1fbernihw77x94pzik71mvn:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 293}, "eval/total_success_rate": 0.550000011920929, "eval/total_success_rate_lb_0.01": 0.3979785144329071, "eval/total_success_rate_ub_0.01": 0.6944345235824585, "eval/total_success_rate_lb_0.025": 0.4265613257884979, "eval/total_success_rate_ub_0.025": 0.6780518889427185, "eval/total_success_rate_lb_0.05": 0.4461565911769867, "eval/total_success_rate_ub_0.05": 0.6503903865814209, "eval/total_success_rate_lb_0.1": 0.46218013763427734, "eval/total_success_rate_ub_0.1": 0.6364701986312866}
\ No newline at end of file
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/logs/debug-internal.log b/8511ppcj/wandb/run-20250101_114617-8511ppcj/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..67493b0f2bbbc13eda616bdd93f410df70f6663f
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/logs/debug-internal.log
@@ -0,0 +1,803 @@
+2025-01-01 11:46:17,424 INFO    StreamThr :1797655 [internal.py:wandb_internal():85] W&B internal server running at pid: 1797655, started at: 2025-01-01 11:46:17.423856
+2025-01-01 11:46:17,425 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status
+2025-01-01 11:46:17,437 INFO    WriterThread:1797655 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/run-8511ppcj.wandb
+2025-01-01 11:46:17,437 DEBUG   SenderThread:1797655 [sender.py:send():379] send: header
+2025-01-01 11:46:17,443 DEBUG   SenderThread:1797655 [sender.py:send():379] send: run
+2025-01-01 11:46:17,681 INFO    SenderThread:1797655 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files
+2025-01-01 11:46:17,681 INFO    SenderThread:1797655 [sender.py:_start_run_threads():1188] run started: 8511ppcj with start time 1735749977.424992
+2025-01-01 11:46:17,698 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: check_version
+2025-01-01 11:46:17,698 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: check_version
+2025-01-01 11:46:17,772 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: run_start
+2025-01-01 11:46:17,776 DEBUG   HandlerThread:1797655 [system_info.py:__init__():26] System info init
+2025-01-01 11:46:17,776 DEBUG   HandlerThread:1797655 [system_info.py:__init__():41] System info init done
+2025-01-01 11:46:17,776 INFO    HandlerThread:1797655 [system_monitor.py:start():194] Starting system monitor
+2025-01-01 11:46:17,776 INFO    SystemMonitor:1797655 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-01 11:46:17,777 INFO    HandlerThread:1797655 [system_monitor.py:probe():214] Collecting system info
+2025-01-01 11:46:17,777 INFO    SystemMonitor:1797655 [interfaces.py:start():188] Started cpu monitoring
+2025-01-01 11:46:17,777 INFO    SystemMonitor:1797655 [interfaces.py:start():188] Started disk monitoring
+2025-01-01 11:46:17,778 INFO    SystemMonitor:1797655 [interfaces.py:start():188] Started gpu monitoring
+2025-01-01 11:46:17,779 INFO    SystemMonitor:1797655 [interfaces.py:start():188] Started memory monitoring
+2025-01-01 11:46:17,779 INFO    SystemMonitor:1797655 [interfaces.py:start():188] Started network monitoring
+2025-01-01 11:46:17,789 DEBUG   HandlerThread:1797655 [system_info.py:probe():152] Probing system
+2025-01-01 11:46:17,790 DEBUG   HandlerThread:1797655 [system_info.py:_probe_git():137] Probing git
+2025-01-01 11:46:17,795 DEBUG   HandlerThread:1797655 [system_info.py:_probe_git():145] Probing git done
+2025-01-01 11:46:17,795 DEBUG   HandlerThread:1797655 [system_info.py:probe():200] Probing system done
+2025-01-01 11:46:17,795 DEBUG   HandlerThread:1797655 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:46:17.789371', 'startedAt': '2025-01-01T16:46:17.415416', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3751.5314999999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4505.388, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 724.0196418762207}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-01 11:46:17,795 INFO    HandlerThread:1797655 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-01 11:46:17,795 INFO    HandlerThread:1797655 [system_monitor.py:probe():227] Publishing system info
+2025-01-01 11:46:17,795 DEBUG   HandlerThread:1797655 [system_info.py:_save_code():44] Saving code
+2025-01-01 11:46:17,796 DEBUG   HandlerThread:1797655 [system_info.py:_save_code():67] Saving code done
+2025-01-01 11:46:17,796 DEBUG   HandlerThread:1797655 [system_info.py:_save_patches():84] Saving git patches
+2025-01-01 11:46:17,858 DEBUG   HandlerThread:1797655 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-01 11:46:17,858 INFO    HandlerThread:1797655 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-01 11:46:17,862 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:46:17,862 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-01 11:46:17,862 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-01 11:46:17,862 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-01 11:46:18,094 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-01 11:46:18,094 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:46:18,094 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: python_packages
+2025-01-01 11:46:18,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:46:18,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:18,097 INFO    HandlerThread:1797655 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/tb"
+  save: true
+}
+control {
+  relay_id: "19a8f7b8037f4a2fb815168a54265f74"
+}
+_info {
+  stream_id: "8511ppcj"
+}
+
+2025-01-01 11:46:18,097 DEBUG   HandlerThread:1797655 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-01 11:46:18,123 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/7yjhb4lx-wandb-metadata.json
+2025-01-01 11:46:18,157 DEBUG   SenderThread:1797655 [sender.py:send():379] send: telemetry
+2025-01-01 11:46:18,157 DEBUG   SenderThread:1797655 [sender.py:send():379] send: tbrecord
+2025-01-01 11:46:18,158 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/i7bjapkn-diff.patch
+2025-01-01 11:46:18,158 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:46:18,158 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0 with policy live
+2025-01-01 11:46:18,189 INFO    wandb-upload_1:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/lygq3cfw-code/FISH/eval_robot.py
+2025-01-01 11:46:18,342 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/uxgn6aox-events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:46:18,683 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:46:18,683 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code/FISH/eval_robot.py
+2025-01-01 11:46:18,683 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:18,683 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/diff.patch
+2025-01-01 11:46:18,683 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/requirements.txt
+2025-01-01 11:46:18,684 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-metadata.json
+2025-01-01 11:46:18,684 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code
+2025-01-01 11:46:18,684 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code/FISH
+2025-01-01 11:46:19,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:20,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:20,682 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:21,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:22,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:23,057 DEBUG   SenderThread:1797655 [sender.py:send():379] send: config
+2025-01-01 11:46:23,057 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:23,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:24,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:24,684 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:25,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:26,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:26,684 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:27,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:28,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:28,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:28,685 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:29,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:30,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:30,685 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:31,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:32,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:32,686 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:33,094 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:46:33,094 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:46:33,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:33,181 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:34,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:34,687 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:35,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:36,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:36,687 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:37,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:38,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:38,688 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:39,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:39,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:40,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:41,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:42,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:43,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:44,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:44,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:45,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:46,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:47,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:48,094 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:46:48,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:46:48,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:49,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:49,226 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:49,691 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml
+2025-01-01 11:46:50,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:51,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:52,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:53,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:54,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:55,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:46:55,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:56,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:56,693 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:46:57,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:58,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:46:59,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:00,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:00,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:01,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:02,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:03,094 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:47:03,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:47:03,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:04,101 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:04,383 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:47:04,383 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_28199d60beae23009631.mp4 with policy now
+2025-01-01 11:47:04,384 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:47:04,384 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:47:04,385 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:47:04,385 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:47:04,641 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/bsau7ev0-media/videos/eval/0_eval_0_28199d60beae23009631.mp4
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/0_eval_0_28199d60beae23009631.mp4
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos
+2025-01-01 11:47:04,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval
+2025-01-01 11:47:04,953 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/wu0d40av-media/videos/eval/0_eval_0_28199d60beae23009631.mp4
+2025-01-01 11:47:05,101 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:05,394 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:06,101 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:06,695 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:07,101 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:08,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:09,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:10,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:10,437 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:10,697 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:11,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:12,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:12,698 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:13,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:14,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:15,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:16,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:16,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:17,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:17,780 DEBUG   SystemMonitor:1797655 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-01 11:47:17,781 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:47:18,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:47:18,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:47:18,141 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:19,103 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:20,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:21,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:21,216 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:22,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:23,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:24,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:25,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:26,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:27,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:27,104 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:28,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:29,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:30,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:31,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:32,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:32,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:33,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:47:33,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:47:33,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:34,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:35,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:35,705 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:47:35,944 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/ssnj77ue-events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:47:36,105 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:36,705 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:37,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:37,449 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:38,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:38,706 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:39,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:40,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:40,707 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:47:41,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:42,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:43,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:43,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:44,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:45,106 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:46,107 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:47,107 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:47,781 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:47:48,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:47:48,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:47:48,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:48,245 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:49,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:50,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:51,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:52,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:53,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:54,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:54,109 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:55,109 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:56,109 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:57,109 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:58,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:47:59,098 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:47:59,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:00,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:01,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:02,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:03,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:48:03,095 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:48:03,141 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:04,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:04,202 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:05,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:06,113 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:07,113 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:07,444 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:48:07,444 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4 with policy now
+2025-01-01 11:48:07,445 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:48:07,445 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:48:07,445 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:48:07,446 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:48:07,705 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/hpl97859-media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4
+2025-01-01 11:48:07,715 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:48:07,715 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4
+2025-01-01 11:48:07,716 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval
+2025-01-01 11:48:07,999 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/27flny6g-media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4
+2025-01-01 11:48:08,113 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:08,716 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:09,113 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:09,716 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:10,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:10,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:11,116 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:12,113 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:12,717 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:13,114 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:14,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:15,119 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:15,409 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:16,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:16,718 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:17,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:17,782 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:48:18,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:48:18,096 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:48:18,141 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:19,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:20,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:21,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:21,157 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:22,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:23,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:23,525 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:48:23,526 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:48:23,527 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:48:23,721 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:48:23,721 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:48:24,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:24,721 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:25,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:26,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:26,483 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:26,722 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:27,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:28,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:28,722 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:29,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:30,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:31,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:32,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:32,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:33,095 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:48:33,096 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:48:33,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:34,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:35,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:36,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:37,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:37,223 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:38,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:39,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:40,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:41,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:42,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:43,100 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:43,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:44,123 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:45,124 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:46,124 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:47,124 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:47,782 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:48:48,096 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:48:48,096 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:48:48,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:48,186 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:49,125 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:50,125 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:51,125 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:52,125 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:52,730 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:53,125 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:53,424 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:54,128 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:54,659 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:48:54,659 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4 with policy now
+2025-01-01 11:48:54,660 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:48:54,660 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:48:54,660 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:48:54,661 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:48:54,731 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:48:54,731 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:54,731 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4
+2025-01-01 11:48:54,731 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval
+2025-01-01 11:48:54,922 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/2s1mhs27-media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4
+2025-01-01 11:48:55,133 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:55,181 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/e1ielie5-media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4
+2025-01-01 11:48:56,132 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:56,731 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:48:57,133 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:58,144 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:48:58,705 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:48:59,133 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:00,135 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:00,733 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:01,140 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:02,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:02,733 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:03,102 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:49:03,102 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:49:03,145 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:04,134 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:04,171 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:05,133 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:06,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:07,139 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:08,143 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:09,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:10,107 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:10,139 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:11,141 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:12,139 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:13,139 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:14,142 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:15,108 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:15,143 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:16,145 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:17,144 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:17,782 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:49:18,102 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:49:18,103 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:49:18,149 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:19,144 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:20,145 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:20,176 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:21,149 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:22,149 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:22,372 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:49:22,373 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:49:22,373 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:49:22,740 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:49:22,740 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:49:22,989 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/liwo1zvs-events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:49:23,148 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:24,152 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:24,741 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:25,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:25,277 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:26,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:27,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:28,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:28,742 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:29,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:30,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:30,743 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:30,990 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:31,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:32,150 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:33,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:49:33,098 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:49:33,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:34,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:35,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:36,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:36,239 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:37,155 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:38,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:39,151 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:40,152 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:41,152 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:42,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:42,152 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:43,152 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:44,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:45,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:46,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:47,110 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:47,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:47,783 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:49:48,097 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:49:48,098 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:49:48,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:49,153 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:50,154 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:51,154 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:52,154 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:52,212 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:53,173 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:54,156 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:54,281 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:49:54,281 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4 with policy now
+2025-01-01 11:49:54,289 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:49:54,290 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:49:54,290 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:49:54,291 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:49:54,578 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/qhh694eq-media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4
+2025-01-01 11:49:54,751 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:49:54,751 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:54,751 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4
+2025-01-01 11:49:54,752 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval
+2025-01-01 11:49:55,008 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/18v4tl5l-media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4
+2025-01-01 11:49:55,160 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:56,175 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:56,752 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:49:57,165 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:57,323 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:49:58,166 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:49:59,168 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:00,169 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:00,753 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:01,173 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:02,168 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:02,754 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:03,101 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:50:03,102 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:50:03,167 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:03,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:04,166 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:05,175 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:06,177 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:07,178 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:08,172 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:09,121 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:09,178 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:10,179 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:11,180 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:12,182 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:13,185 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:14,122 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:14,187 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:15,189 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:16,188 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:17,186 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:17,783 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:50:18,137 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:50:18,138 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:50:18,189 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:18,439 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:50:18,439 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:50:18,439 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:50:18,759 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:50:18,759 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:50:19,155 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:19,197 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:20,192 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:20,760 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:21,198 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:22,194 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:23,217 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:24,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:24,440 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:24,761 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:25,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:26,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:27,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:28,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:29,193 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:30,114 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:30,194 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:31,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:32,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:33,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:50:33,099 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:50:33,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:34,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:35,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:35,254 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:36,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:37,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:38,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:38,766 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:39,195 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:40,196 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:41,118 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:41,196 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:42,196 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:43,196 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:44,196 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:45,199 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:46,197 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:47,118 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:47,197 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:47,784 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:50:48,099 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:50:48,099 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:50:48,197 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:49,198 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:50,198 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:51,204 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:51,529 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:50:51,529 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4 with policy now
+2025-01-01 11:50:51,535 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:50:51,536 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:50:51,536 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:50:51,537 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:50:51,770 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:50:51,770 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4
+2025-01-01 11:50:51,771 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval
+2025-01-01 11:50:51,790 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/0iu5p2o0-media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4
+2025-01-01 11:50:52,050 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/j32q2l82-media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4
+2025-01-01 11:50:52,202 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:52,556 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:52,771 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:53,206 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:53,771 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:54,214 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:55,206 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:56,213 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:56,772 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:50:57,203 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:57,605 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:50:58,225 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:50:59,216 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:00,215 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:00,773 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:01,204 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:02,209 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:03,116 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-01 11:51:03,116 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: stop_status
+2025-01-01 11:51:03,208 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:51:03,209 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:04,211 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:05,222 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:06,222 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:07,222 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:08,214 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:08,499 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:51:08,499 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:08,499 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:51:08,499 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:08,776 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:51:08,776 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:51:08,776 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:08,913 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-01 11:51:08,913 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: log_artifact
+2025-01-01 11:51:09,061 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/pma5ojbp-events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:51:09,120 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:51:09,215 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:09,425 INFO    wandb-upload_0:1797655 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1b9puqx6
+2025-01-01 11:51:09,551 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-01 11:51:09,559 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table
+2025-01-01 11:51:09,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval
+2025-01-01 11:51:09,952 INFO    SenderThread:1797655 [sender.py:send_request_log_artifact():1525] logged artifact run-8511ppcj-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcwMzQ5OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDQ0Mg==', 'latestArtifact': None}}
+2025-01-01 11:51:09,952 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:51:09,952 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2025-01-01 11:51:09,953 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:51:09,953 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,954 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,954 DEBUG   SenderThread:1797655 [sender.py:send():379] send: telemetry
+2025-01-01 11:51:09,954 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,958 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,958 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,958 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,958 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,959 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,959 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,959 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,959 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,960 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,960 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,960 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,960 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,961 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,961 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,961 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,961 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,961 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,962 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,962 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,962 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:09,962 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:09,963 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: log_artifact
+2025-01-01 11:51:10,126 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/7hxr9auz-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-01 11:51:10,167 INFO    wandb-upload_1:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/79iihm2m-media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json
+2025-01-01 11:51:10,215 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:10,332 INFO    wandb-upload_0:1797655 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp7k2ccjgz
+2025-01-01 11:51:10,371 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/tzw4ycwg-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-01 11:51:10,713 INFO    SenderThread:1797655 [sender.py:send_request_log_artifact():1525] logged artifact run-8511ppcj-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDcwMzUyMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0NDQ0Nw==', 'latestArtifact': None}}
+2025-01-01 11:51:10,714 DEBUG   SenderThread:1797655 [sender.py:send():379] send: files
+2025-01-01 11:51:10,714 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json with policy now
+2025-01-01 11:51:10,714 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:51:10,714 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:10,715 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:10,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:51:10,777 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:10,984 INFO    wandb-upload_1:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/cn7k02se-media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json
+2025-01-01 11:51:11,215 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:11,721 DEBUG   SenderThread:1797655 [sender.py:send():379] send: exit
+2025-01-01 11:51:11,722 INFO    SenderThread:1797655 [sender.py:send_exit():586] handling exit code: 0
+2025-01-01 11:51:11,722 INFO    SenderThread:1797655 [sender.py:send_exit():588] handling runtime: 293
+2025-01-01 11:51:11,722 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:11,722 INFO    SenderThread:1797655 [sender.py:send_exit():594] send defer
+2025-01-01 11:51:11,722 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:11,722 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-01 11:51:11,722 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:11,722 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-01 11:51:11,723 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 1
+2025-01-01 11:51:11,723 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:11,723 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-01 11:51:11,723 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:11,723 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-01 11:51:11,723 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 2
+2025-01-01 11:51:11,723 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:11,723 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-01 11:51:11,723 INFO    HandlerThread:1797655 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-01 11:51:11,723 DEBUG   SystemMonitor:1797655 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-01 11:51:11,723 DEBUG   SystemMonitor:1797655 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-01 11:51:11,724 INFO    HandlerThread:1797655 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-01 11:51:11,724 INFO    HandlerThread:1797655 [interfaces.py:finish():200] Joined disk monitor
+2025-01-01 11:51:11,777 INFO    HandlerThread:1797655 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-01 11:51:11,777 INFO    HandlerThread:1797655 [interfaces.py:finish():200] Joined memory monitor
+2025-01-01 11:51:11,777 INFO    HandlerThread:1797655 [interfaces.py:finish():200] Joined network monitor
+2025-01-01 11:51:11,778 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:11,778 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-01 11:51:11,778 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 3
+2025-01-01 11:51:11,778 DEBUG   SenderThread:1797655 [sender.py:send():379] send: stats
+2025-01-01 11:51:11,778 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:11,778 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-01 11:51:11,778 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:11,778 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-01 11:51:11,778 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 4
+2025-01-01 11:51:11,779 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:11,779 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-01 11:51:11,779 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:51:12,780 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:18,504 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:51:18,504 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:18,505 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:51:18,505 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-01 11:51:18,505 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-01 11:51:18,505 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 5
+2025-01-01 11:51:18,506 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:51:18,506 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:18,506 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-01 11:51:18,506 DEBUG   SenderThread:1797655 [sender.py:send():379] send: history
+2025-01-01 11:51:18,507 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: summary_record
+2025-01-01 11:51:18,507 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:18,507 DEBUG   SenderThread:1797655 [sender.py:send():379] send: summary
+2025-01-01 11:51:18,508 INFO    SenderThread:1797655 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-01 11:51:18,508 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:18,508 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-01 11:51:18,508 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 6
+2025-01-01 11:51:18,508 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:18,508 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-01 11:51:18,509 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:18,509 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-01 11:51:18,512 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: status_report
+2025-01-01 11:51:18,635 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 7
+2025-01-01 11:51:18,636 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:18,636 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-01 11:51:18,636 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:18,636 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-01 11:51:18,723 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:51:18,782 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:51:18,782 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml
+2025-01-01 11:51:21,623 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 8
+2025-01-01 11:51:21,623 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:51:21,624 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:21,624 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-01 11:51:21,624 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:21,624 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-01 11:51:21,624 INFO    SenderThread:1797655 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-01 11:51:21,626 INFO    SenderThread:1797655 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-01 11:51:21,628 INFO    SenderThread:1797655 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-01 11:51:21,637 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 9
+2025-01-01 11:51:21,637 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:21,637 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-01 11:51:21,637 DEBUG   SenderThread:1797655 [sender.py:send():379] send: artifact
+2025-01-01 11:51:21,724 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:51:21,783 INFO    Thread-12 :1797655 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:21,816 INFO    SenderThread:1797655 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}}
+2025-01-01 11:51:21,816 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:21,816 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-01 11:51:21,816 INFO    SenderThread:1797655 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-01 11:51:22,784 INFO    SenderThread:1797655 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files
+2025-01-01 11:51:22,784 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml config.yaml
+2025-01-01 11:51:22,784 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json wandb-summary.json
+2025-01-01 11:51:22,784 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/requirements.txt requirements.txt
+2025-01-01 11:51:22,785 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-metadata.json wandb-metadata.json
+2025-01-01 11:51:22,785 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0 events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log output.log
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/diff.patch diff.patch
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/0_eval_0_28199d60beae23009631.mp4 media/videos/eval/0_eval_0_28199d60beae23009631.mp4
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4 media/videos/eval/4_eval_4_77c14f84a9e39e1d85ee.mp4
+2025-01-01 11:51:22,790 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4 media/videos/eval/1_eval_1_c50442085fb847ef468d.mp4
+2025-01-01 11:51:22,791 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4 media/videos/eval/3_eval_3_c9555eba9b6b5e518c21.mp4
+2025-01-01 11:51:22,791 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4 media/videos/eval/2_eval_2_07e03518140cc7a8fdab.mp4
+2025-01-01 11:51:22,793 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json media/table/eval/total_success_rate_ci_6_101bd877386d563fcdb4.table.json
+2025-01-01 11:51:22,794 INFO    SenderThread:1797655 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-01 11:51:22,794 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 10
+2025-01-01 11:51:22,794 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:51:22,794 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:22,794 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-01 11:51:22,794 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:22,794 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-01 11:51:22,794 INFO    SenderThread:1797655 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-01 11:51:22,968 INFO    wandb-upload_0:1797655 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/wandb-summary.json
+2025-01-01 11:51:22,984 INFO    wandb-upload_2:1797655 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/config.yaml
+2025-01-01 11:51:22,995 INFO    wandb-upload_1:1797655 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/requirements.txt
+2025-01-01 11:51:22,997 INFO    wandb-upload_4:1797655 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/files/output.log
+2025-01-01 11:51:23,093 INFO    wandb-upload_3:1797655 [upload_job.py:push():130] Uploaded file /tmp/tmp42gfm2n1wandb/p1bqef6k-events.out.tfevents.1735749978.leonmkim-ROG-Strix-G15CS-G15CS.1797490.0
+2025-01-01 11:51:23,294 INFO    Thread-11 (_thread_body):1797655 [sender.py:transition_state():614] send defer: 11
+2025-01-01 11:51:23,294 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:23,294 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-01 11:51:23,294 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:23,294 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-01 11:51:23,294 INFO    SenderThread:1797655 [file_pusher.py:join():175] waiting for file pusher
+2025-01-01 11:51:23,295 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 12
+2025-01-01 11:51:23,296 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:23,296 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-01 11:51:23,296 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:23,296 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-01 11:51:23,296 INFO    SenderThread:1797655 [file_stream.py:finish():601] file stream finish called
+2025-01-01 11:51:23,533 INFO    SenderThread:1797655 [file_stream.py:finish():605] file stream finish is done
+2025-01-01 11:51:23,533 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 13
+2025-01-01 11:51:23,534 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:23,534 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-01 11:51:23,534 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:23,534 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-01 11:51:23,534 INFO    SenderThread:1797655 [sender.py:transition_state():614] send defer: 14
+2025-01-01 11:51:23,534 DEBUG   SenderThread:1797655 [sender.py:send():379] send: final
+2025-01-01 11:51:23,534 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: defer
+2025-01-01 11:51:23,534 INFO    HandlerThread:1797655 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-01 11:51:23,534 DEBUG   SenderThread:1797655 [sender.py:send():379] send: footer
+2025-01-01 11:51:23,535 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: defer
+2025-01-01 11:51:23,535 INFO    SenderThread:1797655 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-01 11:51:23,535 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:51:23,535 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: server_info
+2025-01-01 11:51:23,535 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-01 11:51:23,536 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:51:23,536 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-01 11:51:23,536 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: server_info
+2025-01-01 11:51:23,536 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-01 11:51:23,538 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-01 11:51:23,586 DEBUG   SenderThread:1797655 [sender.py:send_request():406] send_request: poll_exit
+2025-01-01 11:51:23,586 INFO    MainThread:1797655 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-01 11:51:23,586 INFO    MainThread:1797655 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-01 11:51:23,587 INFO    MainThread:1797655 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-01 11:51:23,587 DEBUG   HandlerThread:1797655 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-01 11:51:23,587 INFO    HandlerThread:1797655 [handler.py:finish():882] shutting down handler
+2025-01-01 11:51:24,536 INFO    WriterThread:1797655 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/run-8511ppcj.wandb
+2025-01-01 11:51:24,586 INFO    SenderThread:1797655 [sender.py:finish():1615] shutting down sender
+2025-01-01 11:51:24,586 INFO    SenderThread:1797655 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-01 11:51:24,586 INFO    SenderThread:1797655 [file_pusher.py:join():175] waiting for file pusher
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/logs/debug.log b/8511ppcj/wandb/run-20250101_114617-8511ppcj/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..ffe5e10d7d63b07805f5db75bc08fa5b9db95a44
--- /dev/null
+++ b/8511ppcj/wandb/run-20250101_114617-8511ppcj/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Configure stats pid to 1797490
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/settings
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/logs/debug.log
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/wandb/run-20250101_114617-8511ppcj/logs/debug-internal.log
+2025-01-01 11:46:17,418 INFO    MainThread:1797490 [wandb_init.py:init():569] calling init triggers
+2025-01-01 11:46:17,419 INFO    MainThread:1797490 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj'}
+2025-01-01 11:46:17,419 INFO    MainThread:1797490 [wandb_init.py:init():619] starting backend
+2025-01-01 11:46:17,419 INFO    MainThread:1797490 [wandb_init.py:init():623] setting up manager
+2025-01-01 11:46:17,423 INFO    MainThread:1797490 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-01 11:46:17,424 INFO    MainThread:1797490 [wandb_init.py:init():631] backend started and connected
+2025-01-01 11:46:17,436 INFO    MainThread:1797490 [wandb_init.py:init():720] updated telemetry
+2025-01-01 11:46:17,441 INFO    MainThread:1797490 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-01 11:46:17,697 INFO    MainThread:1797490 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-01 11:46:17,767 INFO    MainThread:1797490 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-01 11:46:17,767 INFO    MainThread:1797490 [wandb_init.py:init():804] starting run threads in backend
+2025-01-01 11:46:18,095 INFO    MainThread:1797490 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-01 11:46:18,095 INFO    MainThread:1797490 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-01 11:46:18,095 INFO    MainThread:1797490 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-01 11:46:18,095 INFO    MainThread:1797490 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-01 11:46:18,096 INFO    MainThread:1797490 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-01 11:46:18,097 INFO    MainThread:1797490 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/8511ppcj/tb, True
+2025-01-01 11:46:23,056 INFO    MainThread:1797490 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'}
+2025-01-01 11:51:24,587 WARNING MsgRouterThr:1797490 [router.py:message_loop():77] message_loop has been closed
diff --git a/8511ppcj/wandb/run-20250101_114617-8511ppcj/run-8511ppcj.wandb b/8511ppcj/wandb/run-20250101_114617-8511ppcj/run-8511ppcj.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..b6642ae06ec7269da1b4f3987e6f7b6a30797ae9
Binary files /dev/null and b/8511ppcj/wandb/run-20250101_114617-8511ppcj/run-8511ppcj.wandb differ
diff --git a/9rj28b8z/.hydra/config.yaml b/9rj28b8z/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe
--- /dev/null
+++ b/9rj28b8z/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1007_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/9rj28b8z/.hydra/hydra.yaml b/9rj28b8z/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8cfdd1fbdb8ca609233e4bf2de31d31e9ee9bf4b
--- /dev/null
+++ b/9rj28b8z/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/9rj28b8z/.hydra/overrides.yaml b/9rj28b8z/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/9rj28b8z/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/9rj28b8z/episode_rosbags/aligned_depth_to_color_K.npy b/9rj28b8z/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/9rj28b8z/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/9rj28b8z/episode_rosbags/cam_tf_world.npy b/9rj28b8z/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/9rj28b8z/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/9rj28b8z/episode_rosbags/color_K.npy b/9rj28b8z/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/9rj28b8z/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/9rj28b8z/episode_rosbags/depth_K.npy b/9rj28b8z/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/9rj28b8z/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/9rj28b8z/eval_robot.log b/9rj28b8z/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..0c5f5ce8f25584023296131e75f0e476bbde0a8a
--- /dev/null
+++ b/9rj28b8z/eval_robot.log
@@ -0,0 +1,12 @@
+[2024-12-31 22:20:40,403][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-31 22:20:40,406][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/9rj28b8z/eval_video/0_eval.mp4 b/9rj28b8z/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ce7c5567ea26c65c5b13f84953300d7c57bb6dee
--- /dev/null
+++ b/9rj28b8z/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4375a602e73826f9f16a54a6fdcc942e4d5ff4735511940bd47468e2cf34ae61
+size 1136811
diff --git a/9rj28b8z/eval_video/1_eval.mp4 b/9rj28b8z/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5024bfa2692013539fe97a299064dcebbe52a110
--- /dev/null
+++ b/9rj28b8z/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1adf501ad4a659c079d001b40c52c961be78102f51bb49d3b0e8589f66d698b
+size 1137671
diff --git a/9rj28b8z/eval_video/2_eval.mp4 b/9rj28b8z/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9ef4c35a815d6d6ad24763e3187c3a9a97b9f27f
--- /dev/null
+++ b/9rj28b8z/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2496e43e4bbc06af2a2a74a6f4c08c8636fb556602d2518fa509a82e9308d388
+size 1137719
diff --git a/9rj28b8z/eval_video/3_eval.mp4 b/9rj28b8z/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..81ee7ded7fbc4095a1b31519072e4e438f112a2d
--- /dev/null
+++ b/9rj28b8z/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ad55986d23b06cab9722d568d32709d3a2da35148da97a66e6534915bfba152
+size 1147929
diff --git a/9rj28b8z/eval_video/4_eval.mp4 b/9rj28b8z/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..41dbb3090756d76da682cf4620aacdbd607177e5
--- /dev/null
+++ b/9rj28b8z/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:209d1fd47e66e5a5c21c994c0d5e7f7ee71f066633add2010a05efb6b88f7c61
+size 1145179
diff --git a/9rj28b8z/tb/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0 b/9rj28b8z/tb/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
new file mode 100644
index 0000000000000000000000000000000000000000..1bd2e2a4848f4cde8bc2c281d9632c9451546add
--- /dev/null
+++ b/9rj28b8z/tb/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:542aa7c7ac46cc7b7ecb73fe9c4363cd7c8cbd32e7d591d085259a617505176d
+size 6539
diff --git a/9rj28b8z/wandb/debug-internal.log b/9rj28b8z/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..baa5d60a6ca4c758d27288e6591036443c4598d7
--- /dev/null
+++ b/9rj28b8z/wandb/debug-internal.log
@@ -0,0 +1,786 @@
+2024-12-31 22:20:46,583 INFO    StreamThr :1756254 [internal.py:wandb_internal():85] W&B internal server running at pid: 1756254, started at: 2024-12-31 22:20:46.582597
+2024-12-31 22:20:46,583 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status
+2024-12-31 22:20:46,599 INFO    WriterThread:1756254 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/run-9rj28b8z.wandb
+2024-12-31 22:20:46,600 DEBUG   SenderThread:1756254 [sender.py:send():379] send: header
+2024-12-31 22:20:46,607 DEBUG   SenderThread:1756254 [sender.py:send():379] send: run
+2024-12-31 22:20:46,848 INFO    SenderThread:1756254 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files
+2024-12-31 22:20:46,848 INFO    SenderThread:1756254 [sender.py:_start_run_threads():1188] run started: 9rj28b8z with start time 1735701646.585916
+2024-12-31 22:20:46,867 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: check_version
+2024-12-31 22:20:46,867 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: check_version
+2024-12-31 22:20:46,944 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: run_start
+2024-12-31 22:20:46,949 DEBUG   HandlerThread:1756254 [system_info.py:__init__():26] System info init
+2024-12-31 22:20:46,949 DEBUG   HandlerThread:1756254 [system_info.py:__init__():41] System info init done
+2024-12-31 22:20:46,949 INFO    HandlerThread:1756254 [system_monitor.py:start():194] Starting system monitor
+2024-12-31 22:20:46,949 INFO    SystemMonitor:1756254 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-31 22:20:46,949 INFO    HandlerThread:1756254 [system_monitor.py:probe():214] Collecting system info
+2024-12-31 22:20:46,950 INFO    SystemMonitor:1756254 [interfaces.py:start():188] Started cpu monitoring
+2024-12-31 22:20:46,951 INFO    SystemMonitor:1756254 [interfaces.py:start():188] Started disk monitoring
+2024-12-31 22:20:46,951 INFO    SystemMonitor:1756254 [interfaces.py:start():188] Started gpu monitoring
+2024-12-31 22:20:46,951 INFO    SystemMonitor:1756254 [interfaces.py:start():188] Started memory monitoring
+2024-12-31 22:20:46,952 INFO    SystemMonitor:1756254 [interfaces.py:start():188] Started network monitoring
+2024-12-31 22:20:46,962 DEBUG   HandlerThread:1756254 [system_info.py:probe():152] Probing system
+2024-12-31 22:20:46,963 DEBUG   HandlerThread:1756254 [system_info.py:_probe_git():137] Probing git
+2024-12-31 22:20:46,968 DEBUG   HandlerThread:1756254 [system_info.py:_probe_git():145] Probing git done
+2024-12-31 22:20:46,968 DEBUG   HandlerThread:1756254 [system_info.py:probe():200] Probing system done
+2024-12-31 22:20:46,968 DEBUG   HandlerThread:1756254 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T03:20:46.962083', 'startedAt': '2025-01-01T03:20:46.569010', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.501, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.948, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.902, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 541.6344528198242}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-31 22:20:46,968 INFO    HandlerThread:1756254 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-31 22:20:46,968 INFO    HandlerThread:1756254 [system_monitor.py:probe():227] Publishing system info
+2024-12-31 22:20:46,968 DEBUG   HandlerThread:1756254 [system_info.py:_save_code():44] Saving code
+2024-12-31 22:20:46,970 DEBUG   HandlerThread:1756254 [system_info.py:_save_code():67] Saving code done
+2024-12-31 22:20:46,970 DEBUG   HandlerThread:1756254 [system_info.py:_save_patches():84] Saving git patches
+2024-12-31 22:20:47,032 DEBUG   HandlerThread:1756254 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-31 22:20:47,033 INFO    HandlerThread:1756254 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-31 22:20:47,036 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:20:47,036 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-31 22:20:47,036 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-31 22:20:47,036 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-31 22:20:47,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-31 22:20:47,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:20:47,277 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: python_packages
+2024-12-31 22:20:47,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:47,277 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:20:47,291 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/afq0b13k-wandb-metadata.json
+2024-12-31 22:20:47,308 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/2fskxp7n-diff.patch
+2024-12-31 22:20:47,321 INFO    HandlerThread:1756254 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/tb"
+  save: true
+}
+control {
+  relay_id: "5141a9007cc4479284808596a6f2d1f2"
+}
+_info {
+  stream_id: "9rj28b8z"
+}
+
+2024-12-31 22:20:47,321 DEBUG   HandlerThread:1756254 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-31 22:20:47,355 DEBUG   SenderThread:1756254 [sender.py:send():379] send: telemetry
+2024-12-31 22:20:47,373 DEBUG   SenderThread:1756254 [sender.py:send():379] send: tbrecord
+2024-12-31 22:20:47,374 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:20:47,374 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0 with policy live
+2024-12-31 22:20:47,423 INFO    wandb-upload_1:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/5v1kig1v-code/FISH/eval_robot.py
+2024-12-31 22:20:47,612 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/1c45muy0-events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/requirements.txt
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code/FISH/eval_robot.py
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-metadata.json
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/diff.patch
+2024-12-31 22:20:47,851 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code
+2024-12-31 22:20:47,851 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code/FISH
+2024-12-31 22:20:48,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:49,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:49,849 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:20:50,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:51,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:52,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:20:52,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:53,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:54,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:55,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:56,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:57,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:20:57,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:58,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:59,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:00,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:01,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:02,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:21:02,277 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:21:02,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:02,391 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:03,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:04,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:05,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:06,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:07,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:08,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:08,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:09,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:10,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:11,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:12,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:13,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:13,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:14,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:15,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:16,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:17,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:21:17,277 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:21:17,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:18,282 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:18,347 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:18,857 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml
+2024-12-31 22:21:19,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:20,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:21,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:22,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:23,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:24,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:24,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:25,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:26,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:27,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:28,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:29,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:29,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:30,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:31,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:31,316 DEBUG   SenderThread:1756254 [sender.py:send():379] send: config
+2024-12-31 22:21:32,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:21:32,277 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:21:32,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:33,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:33,861 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:34,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:34,414 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:35,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:36,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:37,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:37,862 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:38,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:39,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:39,849 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:39,863 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:40,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:41,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:42,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:43,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:44,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:44,867 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:45,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:45,865 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:46,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:46,952 DEBUG   SystemMonitor:1756254 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-31 22:21:46,952 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:21:47,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:21:47,278 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:21:47,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:48,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:49,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:49,866 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:50,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:50,635 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:50,866 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml
+2024-12-31 22:21:51,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:51,867 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:52,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:53,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:54,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:55,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:55,868 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:56,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:56,469 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:57,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:57,868 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:58,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:59,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:59,869 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:00,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:01,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:01,869 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:01,897 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:02,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:22:02,278 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:22:02,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:03,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:04,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:05,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:06,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:07,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:07,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:08,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:09,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:10,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:11,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:12,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:12,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:13,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:14,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:15,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:15,576 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:22:15,576 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4 with policy now
+2024-12-31 22:22:15,576 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:22:15,577 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:22:15,577 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:22:15,577 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos
+2024-12-31 22:22:15,888 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/b0zjmynn-media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4
+2024-12-31 22:22:16,193 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/pbxrlqm2-media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4
+2024-12-31 22:22:16,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:16,953 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:22:17,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:22:17,278 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:22:17,325 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:17,433 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:17,875 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:18,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:19,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:20,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:21,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:21,877 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:22,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:23,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:23,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:23,878 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:24,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:24,878 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:22:25,138 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/zrevcriz-events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:22:25,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:25,878 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:26,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:27,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:27,879 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:28,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:29,007 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:29,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:29,880 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:30,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:31,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:32,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:22:32,278 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:22:32,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:33,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:34,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:34,386 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:35,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:36,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:37,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:38,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:39,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:40,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:40,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:41,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:42,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:43,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:44,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:45,295 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:46,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:46,295 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:46,953 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:22:47,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:22:47,280 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:22:47,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:48,295 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:49,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:50,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:51,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:51,430 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:52,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:53,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:54,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:55,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:55,888 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:56,078 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:22:56,078 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4 with policy now
+2024-12-31 22:22:56,083 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:22:56,084 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:22:56,084 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:22:56,085 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:22:56,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:56,341 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/1kh2rpcb-media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4
+2024-12-31 22:22:56,888 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:22:56,888 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4
+2024-12-31 22:22:56,889 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval
+2024-12-31 22:22:57,093 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:57,190 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/xa3bjmsq-media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4
+2024-12-31 22:22:57,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:57,889 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:58,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:58,889 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:59,305 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:00,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:01,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:01,890 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:02,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:23:02,281 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:23:02,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:02,381 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:03,299 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:03,891 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:04,300 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:05,300 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:05,891 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:23:06,300 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:06,754 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:23:06,755 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:23:06,756 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:23:06,892 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:23:07,300 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:07,892 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:08,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:08,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:09,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:10,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:11,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:11,894 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:12,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:13,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:13,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:14,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:15,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:16,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:16,954 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:23:17,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:23:17,279 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:23:17,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:18,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:18,386 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:19,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:20,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:21,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:22,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:23,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:24,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:24,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:25,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:26,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:27,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:28,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:29,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:30,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:30,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:31,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:32,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:23:32,279 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:23:32,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:33,304 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:34,304 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:35,304 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:35,436 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:36,304 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:37,307 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:37,658 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:23:37,659 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4 with policy now
+2024-12-31 22:23:37,662 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:23:37,663 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:23:37,663 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:23:37,663 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:23:37,902 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:37,902 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:23:37,902 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4
+2024-12-31 22:23:37,903 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval
+2024-12-31 22:23:37,929 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/ehm68xim-media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4
+2024-12-31 22:23:38,200 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/hgcfn8rn-media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4
+2024-12-31 22:23:38,308 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:38,902 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:39,309 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:39,903 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:40,308 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:40,692 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:40,903 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:41,312 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:42,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:43,312 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:43,904 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:44,310 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:45,310 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:45,905 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:45,908 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:46,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:46,586 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:23:46,586 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:23:46,586 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:23:46,905 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:23:46,905 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:23:46,954 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:23:47,168 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/lwhxb3fm-events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:23:47,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:23:47,287 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:23:47,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:47,905 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:48,318 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:49,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:50,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:51,259 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:51,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:51,907 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:52,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:53,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:54,315 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:55,315 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:56,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:56,315 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:57,315 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:58,315 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:59,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:59,910 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:00,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:01,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:01,834 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:02,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:24:02,281 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:24:02,325 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:03,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:03,911 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:04,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:05,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:06,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:07,021 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:07,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:08,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:09,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:10,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:11,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:12,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:12,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:13,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:14,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:15,319 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:16,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:16,955 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:24:17,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:24:17,286 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:24:17,323 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:17,387 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:17,916 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:18,325 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:18,329 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:24:18,329 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4 with policy now
+2024-12-31 22:24:18,377 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:24:18,378 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:24:18,378 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:24:18,378 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:24:18,595 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/95t011jx-media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4
+2024-12-31 22:24:18,917 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:24:18,917 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4
+2024-12-31 22:24:18,917 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval
+2024-12-31 22:24:19,192 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/aai7feos-media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4
+2024-12-31 22:24:19,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:19,917 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:20,336 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:20,917 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:21,325 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:22,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:23,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:23,332 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:23,918 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:24,328 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:25,335 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:25,919 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:26,332 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:27,346 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:28,335 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:29,312 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:29,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:30,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:30,921 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:24:31,338 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:31,642 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:24:31,642 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:24:31,642 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:24:31,921 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:31,921 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:24:32,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:24:32,293 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:24:32,346 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:33,338 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:33,922 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:34,350 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:34,417 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:35,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:35,923 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:36,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:37,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:38,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:39,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:40,308 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:40,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:41,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:42,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:43,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:44,338 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:45,308 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:45,338 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:46,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:46,955 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:24:47,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:24:47,286 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:24:47,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:48,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:49,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:50,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:50,363 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:51,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:52,347 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:53,347 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:54,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:55,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:56,309 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:56,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:57,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:58,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:59,349 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:00,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:01,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:01,781 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:02,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:25:02,294 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:25:02,361 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:02,996 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:25:02,996 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4 with policy now
+2024-12-31 22:25:02,999 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:25:03,007 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:25:03,007 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:03,008 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:03,272 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/r4s241db-media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4
+2024-12-31 22:25:03,356 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:03,932 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:03,933 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:03,933 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4
+2024-12-31 22:25:03,933 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval
+2024-12-31 22:25:04,233 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/yculme8b-media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4
+2024-12-31 22:25:04,369 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:05,364 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:06,379 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:07,312 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:07,372 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:07,934 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:08,369 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:09,469 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:10,473 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:11,472 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:11,935 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:12,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:12,482 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:13,479 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:14,479 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:15,484 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:15,695 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:25:15,695 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:15,696 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:15,937 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:15,937 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:25:16,185 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/ldqqyaau-events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:25:16,481 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:16,567 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:25:16,568 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:25:16,743 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:25:16,938 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:25:16,938 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media
+2024-12-31 22:25:16,938 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table
+2024-12-31 22:25:16,938 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval
+2024-12-31 22:25:17,121 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/4z7x5lq3-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:25:17,121 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:25:17,130 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:25:17,175 INFO    wandb-upload_0:1756254 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp78cj2gg3
+2024-12-31 22:25:17,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:25:17,471 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:17,758 INFO    SenderThread:1756254 [sender.py:send_request_log_artifact():1525] logged artifact run-9rj28b8z-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE1OTU5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MjczOQ==', 'latestArtifact': None}}
+2024-12-31 22:25:17,759 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:25:17,759 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2024-12-31 22:25:17,759 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:25:17,759 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,759 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:17,760 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,760 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:25:17,760 DEBUG   SenderThread:1756254 [sender.py:send():379] send: telemetry
+2024-12-31 22:25:17,760 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,760 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,761 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,761 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,761 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,763 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,763 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,763 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,763 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,764 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,764 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,764 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,764 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,764 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,765 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,765 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,765 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,765 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,765 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,766 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,766 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,766 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,766 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:25:17,938 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:17,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:17,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:25:17,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json
+2024-12-31 22:25:17,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval
+2024-12-31 22:25:18,012 INFO    wandb-upload_1:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/kspg8hf6-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:25:18,178 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/qtrdskjr-media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json
+2024-12-31 22:25:18,273 INFO    wandb-upload_2:1756254 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5_8snr6f
+2024-12-31 22:25:18,471 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:18,774 INFO    SenderThread:1756254 [sender.py:send_request_log_artifact():1525] logged artifact run-9rj28b8z-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE1OTYwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3Mjc0MA==', 'latestArtifact': None}}
+2024-12-31 22:25:18,774 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:25:18,774 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json with policy now
+2024-12-31 22:25:18,774 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:25:18,774 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:18,775 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:18,775 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:25:18,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:19,027 INFO    wandb-upload_1:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/in3hzrf4-media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json
+2024-12-31 22:25:19,257 DEBUG   SenderThread:1756254 [sender.py:send():379] send: exit
+2024-12-31 22:25:19,258 INFO    SenderThread:1756254 [sender.py:send_exit():586] handling exit code: 0
+2024-12-31 22:25:19,258 INFO    SenderThread:1756254 [sender.py:send_exit():588] handling runtime: 272
+2024-12-31 22:25:19,258 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:19,258 INFO    SenderThread:1756254 [sender.py:send_exit():594] send defer
+2024-12-31 22:25:19,258 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:19,258 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-31 22:25:19,258 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:19,259 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-31 22:25:19,259 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 1
+2024-12-31 22:25:19,259 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:19,259 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-31 22:25:19,259 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:19,259 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-31 22:25:19,259 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 2
+2024-12-31 22:25:19,259 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:19,259 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-31 22:25:19,259 INFO    HandlerThread:1756254 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-31 22:25:19,259 DEBUG   SystemMonitor:1756254 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-31 22:25:19,259 DEBUG   SystemMonitor:1756254 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-31 22:25:19,260 INFO    HandlerThread:1756254 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-31 22:25:19,260 INFO    HandlerThread:1756254 [interfaces.py:finish():200] Joined disk monitor
+2024-12-31 22:25:19,312 INFO    HandlerThread:1756254 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-31 22:25:19,312 INFO    HandlerThread:1756254 [interfaces.py:finish():200] Joined memory monitor
+2024-12-31 22:25:19,312 INFO    HandlerThread:1756254 [interfaces.py:finish():200] Joined network monitor
+2024-12-31 22:25:19,313 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:19,313 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-31 22:25:19,313 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 3
+2024-12-31 22:25:19,313 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:25:19,313 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:19,313 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-31 22:25:19,313 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:19,313 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-31 22:25:19,313 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 4
+2024-12-31 22:25:19,313 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:19,313 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-31 22:25:19,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:19,940 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:21,940 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:25,704 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:25,705 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:25,705 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-31 22:25:25,705 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:25,706 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-31 22:25:25,706 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 5
+2024-12-31 22:25:25,706 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:25,706 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:25:25,706 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:25,706 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:25,706 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-31 22:25:25,707 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:25,707 DEBUG   SenderThread:1756254 [sender.py:send():379] send: summary
+2024-12-31 22:25:25,711 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:25,711 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:25,711 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-31 22:25:25,711 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 6
+2024-12-31 22:25:25,711 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:25,711 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-31 22:25:25,711 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:25,711 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-31 22:25:25,715 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:25,826 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 7
+2024-12-31 22:25:25,826 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:25,826 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-31 22:25:25,826 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:25,826 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-31 22:25:25,942 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:25,942 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml
+2024-12-31 22:25:26,259 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:28,222 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 8
+2024-12-31 22:25:28,222 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:28,222 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:28,222 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-31 22:25:28,223 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:28,223 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-31 22:25:28,223 INFO    SenderThread:1756254 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-31 22:25:28,227 INFO    SenderThread:1756254 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-31 22:25:28,229 INFO    SenderThread:1756254 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-31 22:25:28,248 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 9
+2024-12-31 22:25:28,248 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:28,248 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-31 22:25:28,249 DEBUG   SenderThread:1756254 [sender.py:send():379] send: artifact
+2024-12-31 22:25:28,259 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:28,635 INFO    wandb-upload_0:1756254 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmput398yfj
+2024-12-31 22:25:28,636 INFO    wandb-upload_1:1756254 [upload_job.py:push():85] Skipped uploading /tmp/tmp_y59j39j/wandb-job.json
+2024-12-31 22:25:28,760 INFO    wandb-upload_2:1756254 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwld62_lp
+2024-12-31 22:25:28,943 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:29,418 INFO    SenderThread:1756254 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE1OTY3NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE1NjI2NQ==', 'versionIndex': 395}}}
+2024-12-31 22:25:29,418 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:29,418 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-31 22:25:29,418 INFO    SenderThread:1756254 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml config.yaml
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json wandb-summary.json
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/requirements.txt requirements.txt
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-metadata.json wandb-metadata.json
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0 events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:25:29,945 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log output.log
+2024-12-31 22:25:29,948 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/diff.patch diff.patch
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4 media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4 media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4 media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4 media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4 media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4
+2024-12-31 22:25:29,952 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json
+2024-12-31 22:25:29,952 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:25:29,953 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 10
+2024-12-31 22:25:29,953 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:29,953 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:29,954 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-31 22:25:29,954 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:29,954 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-31 22:25:29,954 INFO    SenderThread:1756254 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:25:30,135 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml
+2024-12-31 22:25:30,152 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/requirements.txt
+2024-12-31 22:25:30,159 INFO    wandb-upload_1:1756254 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:30,165 INFO    wandb-upload_4:1756254 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:30,259 INFO    wandb-upload_3:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/q78ua6a5-events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:25:30,260 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:30,260 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:30,460 INFO    Thread-11 (_thread_body):1756254 [sender.py:transition_state():614] send defer: 11
+2024-12-31 22:25:30,460 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:30,460 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-31 22:25:30,460 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:30,460 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-31 22:25:30,461 INFO    SenderThread:1756254 [file_pusher.py:join():175] waiting for file pusher
+2024-12-31 22:25:30,462 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 12
+2024-12-31 22:25:30,462 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:30,462 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-31 22:25:30,462 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:30,462 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-31 22:25:30,462 INFO    SenderThread:1756254 [file_stream.py:finish():601] file stream finish called
+2024-12-31 22:25:30,519 INFO    SenderThread:1756254 [file_stream.py:finish():605] file stream finish is done
+2024-12-31 22:25:30,519 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 13
+2024-12-31 22:25:30,520 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:30,520 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-31 22:25:30,520 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:30,520 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-31 22:25:30,520 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 14
+2024-12-31 22:25:30,520 DEBUG   SenderThread:1756254 [sender.py:send():379] send: final
+2024-12-31 22:25:30,520 DEBUG   SenderThread:1756254 [sender.py:send():379] send: footer
+2024-12-31 22:25:30,520 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:30,520 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-31 22:25:30,521 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:30,521 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-31 22:25:30,521 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:30,521 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:30,522 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: server_info
+2024-12-31 22:25:30,522 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:30,522 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-31 22:25:30,522 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-31 22:25:30,522 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:30,523 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: server_info
+2024-12-31 22:25:30,575 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:30,575 INFO    MainThread:1756254 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-31 22:25:30,576 INFO    MainThread:1756254 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-31 22:25:30,576 INFO    MainThread:1756254 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-31 22:25:30,576 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-31 22:25:30,576 INFO    HandlerThread:1756254 [handler.py:finish():882] shutting down handler
+2024-12-31 22:25:31,523 INFO    WriterThread:1756254 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/run-9rj28b8z.wandb
+2024-12-31 22:25:31,575 INFO    SenderThread:1756254 [sender.py:finish():1615] shutting down sender
+2024-12-31 22:25:31,575 INFO    SenderThread:1756254 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:25:31,575 INFO    SenderThread:1756254 [file_pusher.py:join():175] waiting for file pusher
diff --git a/9rj28b8z/wandb/debug.log b/9rj28b8z/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..d31bf16dc6ebd29de616c8c5e3784ee1996d89a5
--- /dev/null
+++ b/9rj28b8z/wandb/debug.log
@@ -0,0 +1,31 @@
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Configure stats pid to 1756086
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/settings
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/logs/debug.log
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/logs/debug-internal.log
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_init.py:init():569] calling init triggers
+2024-12-31 22:20:46,576 INFO    MainThread:1756086 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z'}
+2024-12-31 22:20:46,576 INFO    MainThread:1756086 [wandb_init.py:init():619] starting backend
+2024-12-31 22:20:46,576 INFO    MainThread:1756086 [wandb_init.py:init():623] setting up manager
+2024-12-31 22:20:46,581 INFO    MainThread:1756086 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-31 22:20:46,585 INFO    MainThread:1756086 [wandb_init.py:init():631] backend started and connected
+2024-12-31 22:20:46,599 INFO    MainThread:1756086 [wandb_init.py:init():720] updated telemetry
+2024-12-31 22:20:46,605 INFO    MainThread:1756086 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-31 22:20:46,867 INFO    MainThread:1756086 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-31 22:20:46,940 INFO    MainThread:1756086 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-31 22:20:46,940 INFO    MainThread:1756086 [wandb_init.py:init():804] starting run threads in backend
+2024-12-31 22:20:47,277 INFO    MainThread:1756086 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-31 22:20:47,278 INFO    MainThread:1756086 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-31 22:20:47,278 INFO    MainThread:1756086 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-31 22:20:47,278 INFO    MainThread:1756086 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-31 22:20:47,280 INFO    MainThread:1756086 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-31 22:20:47,281 INFO    MainThread:1756086 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/tb, True
+2024-12-31 22:21:31,315 INFO    MainThread:1756086 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-31 22:25:31,577 WARNING MsgRouterThr:1756086 [router.py:message_loop():77] message_loop has been closed
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code/FISH/eval_robot.py b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..6f971c04f88c3f951a2321c29af50f00efe5b3a1
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					total_num_episodes += filtered_run.summary_metrics['episode']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7fe1ab80bd215f577cbc416c3b29350ca372342a
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1007_0'
+wandb_notes:
+  desc: null
+  value: 1007_0_restarted_2
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1735701646
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: hbm
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/diff.patch b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..df1b3966b7d19592540b568ba2e88fffb49e0995
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/diff.patch
@@ -0,0 +1,68 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 0f6a9ec..b9e3dfe 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
++# wandb_run_id: '1002_0' 
++
++# all books, 6/20 demos per book
++# RGBD+mask+act history
+ wandb_run_id: '1007_0' 
+ 
++# RGBD+mask+contact(all ftrs)+act history
++# wandb_run_id: '1006_0'
++
++# RGBD+mask+contact(contact+DTC)+act history
++# wandb_run_id: '1745_0'
+ 
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 21e335c..71a5d1c 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -155,7 +155,12 @@ run_id_list = [
+     # '1022_0',
+     # '1023_0',
+     # '1007_0',
+-    '1007_0'
++    # '1007_0'
++]
++
++run_id_list = [
++    # '1006_0',
++    '1745_0'
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py
+index 4179186..6f971c0 100644
+--- a/FISH/eval_robot.py
++++ b/FISH/eval_robot.py
+@@ -478,8 +478,8 @@ class Workspace:
+ 				"summary_metrics.episode": {"$gte": 5},
+ 				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+ 				"state": "finished",
+-				"config.grasped_obj_name": grasped_obj_name,
+-				"config.left_book_slot": left_book_slot,
++				# "config.grasped_obj_name": grasped_obj_name,
++				# "config.left_book_slot": left_book_slot,
+ 			}
+ 
+ 			api = wandb.Api()
+diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py
+index 3302c31..1946eca 100644
+--- a/gym-envs/gym_envs/envs/franka_env.py
++++ b/gym-envs/gym_envs/envs/franka_env.py
+@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env):
+ 
+ 	def mask_obs_callback(self, msg):
+ 		msg = self.possibly_resize_mask_obs(msg)
+-		rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}")
++		# rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}")
+ 		self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg)  
+ 
+ 	def contact_obs_callback(self, msg):
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]}
\ No newline at end of file
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..d2dd1acb69a719f209918aa897b31f41efe9ec6f
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[14, 25, 0.56, 0.325596690212798, 0.7740918099332035, 0.37193104627309614, 0.7546122967687644, 0.4010925144154953, 0.7093886956153755, 0.4198877960603721, 0.6952312998083181]]}
\ No newline at end of file
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4 b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ce7c5567ea26c65c5b13f84953300d7c57bb6dee
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4375a602e73826f9f16a54a6fdcc942e4d5ff4735511940bd47468e2cf34ae61
+size 1136811
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4 b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5024bfa2692013539fe97a299064dcebbe52a110
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1adf501ad4a659c079d001b40c52c961be78102f51bb49d3b0e8589f66d698b
+size 1137671
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4 b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9ef4c35a815d6d6ad24763e3187c3a9a97b9f27f
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2496e43e4bbc06af2a2a74a6f4c08c8636fb556602d2518fa509a82e9308d388
+size 1137719
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4 b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..81ee7ded7fbc4095a1b31519072e4e438f112a2d
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ad55986d23b06cab9722d568d32709d3a2da35148da97a66e6534915bfba152
+size 1147929
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4 b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..41dbb3090756d76da682cf4620aacdbd607177e5
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:209d1fd47e66e5a5c21c994c0d5e7f7ee71f066633add2010a05efb6b88f7c61
+size 1145179
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..416dda5605df0607a4743a7d6f290d42e1c72320
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
@@ -0,0 +1,278 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1735701691.316340]: resetting environment
+[INFO] [1735701691.319610]: cleared current plan
+[INFO] [1735701691.321337]: moving to home
+[INFO] [1735701692.323135]: reached home
+[INFO] [1735701692.324483]: reset action history
+[INFO] [1735701695.084159]: environment reset
+Starting episode 0
+[INFO] [1735701695.086397]: resetting environment
+[INFO] [1735701695.089785]: cleared current plan
+[INFO] [1735701695.090989]: moving to home
+[INFO] [1735701696.093063]: reached home
+[INFO] [1735701696.094443]: reset action history
+[INFO] [1735701698.806861]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1735701703.859764]: resetting environment
+[INFO] [1735701703.862502]: cleared current plan
+[INFO] [1735701703.863256]: moving to home
+[INFO] [1735701704.866575]: reached home
+[INFO] [1735701704.868904]: reset action history
+[INFO] [1735701707.577449]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/episode_rosbags/episode_0_2024-12-31-22-21-47.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1756086.log
+started roslaunch server http://158.130.50.37:46605/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [1756463]
+process[print_text-2]: started with pid [1756464]
+process[rosbag_record-3]: started with pid [1756465]
+started bagging!
+For topic gripper_width: timestamp difference is 112037732 for nearest: 1735701708419951751 - target: 1735701708307914019 at idx 13
+For topic gripper_width: timestamp difference is 113318424 for nearest: 1735701709020023088 - target: 1735701708906704664 at idx 20
+For topic gripper_width: timestamp difference is 113728414 for nearest: 1735701709220022092 - target: 1735701709106293678 at idx 22
+For topic gripper_width: timestamp difference is 114972938 for nearest: 1735701709820029844 - target: 1735701709705056906 at idx 28
+For topic gripper_width: timestamp difference is 116456747 for nearest: 1735701710619998455 - target: 1735701710503541708 at idx 43
+For topic gripper_width: timestamp difference is 112282977 for nearest: 1735701713019999251 - target: 1735701712907716274 at idx 77
+For topic gripper_width: timestamp difference is 112448259 for nearest: 1735701713219998642 - target: 1735701713107550383 at idx 79
+For topic gripper_width: timestamp difference is 113279373 for nearest: 1735701714220002920 - target: 1735701714106723547 at idx 93
+For topic gripper_width: timestamp difference is 114269284 for nearest: 1735701715420030621 - target: 1735701715305761337 at idx 109
+For topic gripper_width: timestamp difference is 116508368 for nearest: 1735701718220002535 - target: 1735701718103494167 at idx 147
+For topic gripper_width: timestamp difference is 117815973 for nearest: 1735701719853370668 - target: 1735701719735554695 at idx 170
+[INFO] [1735701734.594192]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1735701735.356729]: Stored episode 1.
+[INFO] [1735701735.356963]: Saving video...
+[INFO] [1735701735.561062]: Video saved!
+Starting episode 1
+[INFO] [1735701735.578009]: resetting environment
+[INFO] [1735701735.595336]: cleared current plan
+[INFO] [1735701735.595621]: moving to home
+[INFO] [1735701739.097462]: reached home
+[INFO] [1735701739.097905]: reset action history
+[INFO] [1735701741.807927]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735701744.040341]: resetting environment
+[INFO] [1735701744.043231]: cleared current plan
+[INFO] [1735701744.043386]: moving to home
+[INFO] [1735701745.045065]: reached home
+[INFO] [1735701745.045483]: reset action history
+[INFO] [1735701747.753232]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/episode_rosbags/episode_1_2024-12-31-22-22-27.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1756086.log
+started roslaunch server http://158.130.50.37:34745/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [1756574]
+process[print_text-5]: started with pid [1756575]
+process[rosbag_record-6]: started with pid [1756599]
+started bagging!
+[INFO] [1735701774.731759]: Storing episode...
+[rosbag_record-6] killing on exit[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1735701775.500327]: Stored episode 2.
+[INFO] [1735701775.500848]: Saving video...
+[INFO] [1735701776.053079]: Video saved!
+Starting episode 2
+[INFO] [1735701776.082710]: resetting environment
+[INFO] [1735701776.092711]: cleared current plan
+[INFO] [1735701776.092966]: moving to home
+[WARN] [1735701776.072776]: Plan exhausted
+[INFO] [1735701779.595746]: reached home
+[INFO] [1735701779.600740]: reset action history
+[INFO] [1735701782.322246]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735701785.740054]: resetting environment
+[INFO] [1735701785.750870]: cleared current plan
+[INFO] [1735701785.751491]: moving to home
+[INFO] [1735701786.754483]: reached home
+[INFO] [1735701786.755021]: reset action history
+[INFO] [1735701789.463918]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/episode_rosbags/episode_2_2024-12-31-22-23-09.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1756086.log
+started roslaunch server http://158.130.50.37:44557/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [1756725]
+process[print_text-8]: started with pid [1756726]
+process[rosbag_record-9]: started with pid [1756727]
+started bagging!
+[INFO] [1735701816.423002]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit[pub_text-7] killing on exit
+[WARN] [1735701817.593017]: Plan exhausted
+[INFO] [1735701817.212128]: Stored episode 3.
+[INFO] [1735701817.218655]: Saving video...
+[INFO] [1735701817.598099]: Video saved!
+Starting episode 3
+[INFO] [1735701817.665987]: resetting environment
+[INFO] [1735701817.683460]: cleared current plan
+[INFO] [1735701817.690266]: moving to home
+[WARN] [1735701817.634635]: Plan exhausted
+[WARN] [1735701817.673327]: Plan exhausted
+[INFO] [1735701821.206277]: reached home
+[INFO] [1735701821.208680]: reset action history
+[INFO] [1735701824.014319]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735701825.950397]: resetting environment
+[INFO] [1735701825.969151]: cleared current plan
+[INFO] [1735701825.974507]: moving to home
+[INFO] [1735701826.980919]: reached home
+[INFO] [1735701826.983049]: reset action history
+[INFO] [1735701829.770133]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/episode_rosbags/episode_3_2024-12-31-22-23-49.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1756086.log
+started roslaunch server http://158.130.50.37:45329/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [1756893]
+process[print_text-11]: started with pid [1756894]
+process[rosbag_record-12]: started with pid [1756895]
+started bagging!
+For topic gripper_width: timestamp difference is 113020739 for nearest: 1735701837786671957 - target: 1735701837673651218 at idx 116
+For topic gripper_width: timestamp difference is 117012672 for nearest: 1735701842619997196 - target: 1735701842502984524 at idx 182
+For topic gripper_width: timestamp difference is 117201784 for nearest: 1735701842820024708 - target: 1735701842702822924 at idx 184
+For topic gripper_width: timestamp difference is 117362736 for nearest: 1735701843020024060 - target: 1735701842902661324 at idx 186
+[INFO] [1735701856.964482]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit[pub_text-10] killing on exit
+[INFO] [1735701857.773851]: Stored episode 4.
+[INFO] [1735701857.779177]: Saving video...
+[INFO] [1735701858.281495]: Video saved!
+Starting episode 4
+[INFO] [1735701858.341381]: resetting environment
+[INFO] [1735701858.361214]: cleared current plan
+[INFO] [1735701858.361570]: moving to home
+[WARN] [1735701858.312801]: Plan exhausted
+[WARN] [1735701858.353418]: Plan exhausted
+[INFO] [1735701861.864738]: reached home
+[INFO] [1735701861.871197]: reset action history
+[INFO] [1735701864.653128]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735701870.777711]: resetting environment
+[INFO] [1735701870.812109]: cleared current plan
+[INFO] [1735701870.814641]: moving to home
+[INFO] [1735701871.819844]: reached home
+[INFO] [1735701871.821872]: reset action history
+[INFO] [1735701874.602059]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/episode_rosbags/episode_4_2024-12-31-22-24-34.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1756086.log
+started roslaunch server http://158.130.50.37:46217/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [1757026]
+process[print_text-14]: started with pid [1757027]
+process[rosbag_record-15]: started with pid [1757028]
+started bagging!
+[INFO] [1735701901.781168]: Storing episode...
+[rosbag_record-15] killing on exit[print_text-14] killing on exit
+[pub_text-13] killing on exit
+[INFO] [1735701902.553896]: Stored episode 5.
+[INFO] [1735701902.560253]: Saving video...
+[INFO] [1735701902.945423]: Video saved!
+[INFO] [1735701903.000302]: resetting environment
+[INFO] [1735701903.023746]: cleared current plan
+[INFO] [1735701903.027092]: moving to home
+[INFO] [1735701906.540306]: reached home
+[INFO] [1735701906.545054]: reset action history
+[INFO] [1735701909.483481]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/requirements.txt b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-metadata.json b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..95a85b2b6984507b7c37e0d897f6dbf52a2002a5
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-01T03:20:46.962083",
+    "startedAt": "2025-01-01T03:20:46.569010",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3937.501,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.999,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.948,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.902,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 541.6344528198242
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..708f73ec8a6da0a530bdc0dccc2798713ce72c24
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "4375a602e73826f9f16a54a6fdcc942e4d5ff4735511940bd47468e2cf34ae61", "size": 1136811, "path": "media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4"}, "global_step": 664, "_timestamp": 1735701917.029469, "_runtime": 270.44355297088623, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "d1adf501ad4a659c079d001b40c52c961be78102f51bb49d3b0e8589f66d698b", "size": 1137671, "path": "media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4"}, "eval/num_success": 5.0, "episode": 25.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "2496e43e4bbc06af2a2a74a6f4c08c8636fb556602d2518fa509a82e9308d388", "size": 1137719, "path": "media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "3ad55986d23b06cab9722d568d32709d3a2da35148da97a66e6534915bfba152", "size": 1147929, "path": "media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "209d1fd47e66e5a5c21c994c0d5e7f7ee71f066633add2010a05efb6b88f7c61", "size": 1145179, "path": "media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://1ni54ds8nmyky6buuyyh7s9y7l7xyeomai4476sjyw584nvih2ocf38u5gq89ckzg1psq643822ljzo87uclabrw6n2ek3nwxp8v7zaui2i3l4afk7xl92tmbapeicls/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://9quis1j1z09fz9kfuexzkmkzgrrlsnj8or65opdrq7rsxuet6hwnch41sdb23dc4hd0fk1klw2hh48esxsws7exok7qcoyshtsfs4vf6fhyhf6b8l6xsc039qn4b2vit:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 14, "total_num_episodes": 25, "total_success_rate": 0.56, "total_success_rate_lb_0.01": 0.325596690212798, "total_success_rate_ub_0.01": 0.7740918099332035, "total_success_rate_lb_0.025": 0.37193104627309614, "total_success_rate_ub_0.025": 0.7546122967687644, "total_success_rate_lb_0.05": 0.4010925144154953, "total_success_rate_ub_0.05": 0.7093886956153755, "total_success_rate_lb_0.1": 0.4198877960603721, "total_success_rate_ub_0.1": 0.6952312998083181, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "dd484121e9f12436bed35e2df1bd7d0703ee9c1ec9b7e2d31d633a8ba51b85b9", "size": 506, "artifact_path": "wandb-client-artifact://w3gsuiz1paqjt0kssrt9dkrmxhbe3q9tlhofutiqc1uurle7gys4ry78vgpy5o3vbwmqe14jkssqe8x0ufdp2yy2d3qagc9pubalp85yd8t86badzbv3kki8ay7gyeiq/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://41xttonwf0uv3i5js72wh2zxtvjd1a2i85cxwjpbrfh3uivohh390dtxyh2v7gd0qwwsipy75wxtrgvou5l1fybnwads3ie6ev1ahze2t65wbb81jxva5awqlquv9rzr:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 272}, "eval/total_success_rate": 0.5600000023841858, "eval/total_success_rate_lb_0.01": 0.3255966901779175, "eval/total_success_rate_ub_0.01": 0.7740917801856995, "eval/total_success_rate_lb_0.025": 0.3719310462474823, "eval/total_success_rate_ub_0.025": 0.7546122670173645, "eval/total_success_rate_lb_0.05": 0.401092529296875, "eval/total_success_rate_ub_0.05": 0.7093886733055115, "eval/total_success_rate_lb_0.1": 0.41988781094551086, "eval/total_success_rate_ub_0.1": 0.6952313184738159}
\ No newline at end of file
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/logs/debug-internal.log b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..baa5d60a6ca4c758d27288e6591036443c4598d7
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/logs/debug-internal.log
@@ -0,0 +1,786 @@
+2024-12-31 22:20:46,583 INFO    StreamThr :1756254 [internal.py:wandb_internal():85] W&B internal server running at pid: 1756254, started at: 2024-12-31 22:20:46.582597
+2024-12-31 22:20:46,583 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status
+2024-12-31 22:20:46,599 INFO    WriterThread:1756254 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/run-9rj28b8z.wandb
+2024-12-31 22:20:46,600 DEBUG   SenderThread:1756254 [sender.py:send():379] send: header
+2024-12-31 22:20:46,607 DEBUG   SenderThread:1756254 [sender.py:send():379] send: run
+2024-12-31 22:20:46,848 INFO    SenderThread:1756254 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files
+2024-12-31 22:20:46,848 INFO    SenderThread:1756254 [sender.py:_start_run_threads():1188] run started: 9rj28b8z with start time 1735701646.585916
+2024-12-31 22:20:46,867 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: check_version
+2024-12-31 22:20:46,867 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: check_version
+2024-12-31 22:20:46,944 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: run_start
+2024-12-31 22:20:46,949 DEBUG   HandlerThread:1756254 [system_info.py:__init__():26] System info init
+2024-12-31 22:20:46,949 DEBUG   HandlerThread:1756254 [system_info.py:__init__():41] System info init done
+2024-12-31 22:20:46,949 INFO    HandlerThread:1756254 [system_monitor.py:start():194] Starting system monitor
+2024-12-31 22:20:46,949 INFO    SystemMonitor:1756254 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-31 22:20:46,949 INFO    HandlerThread:1756254 [system_monitor.py:probe():214] Collecting system info
+2024-12-31 22:20:46,950 INFO    SystemMonitor:1756254 [interfaces.py:start():188] Started cpu monitoring
+2024-12-31 22:20:46,951 INFO    SystemMonitor:1756254 [interfaces.py:start():188] Started disk monitoring
+2024-12-31 22:20:46,951 INFO    SystemMonitor:1756254 [interfaces.py:start():188] Started gpu monitoring
+2024-12-31 22:20:46,951 INFO    SystemMonitor:1756254 [interfaces.py:start():188] Started memory monitoring
+2024-12-31 22:20:46,952 INFO    SystemMonitor:1756254 [interfaces.py:start():188] Started network monitoring
+2024-12-31 22:20:46,962 DEBUG   HandlerThread:1756254 [system_info.py:probe():152] Probing system
+2024-12-31 22:20:46,963 DEBUG   HandlerThread:1756254 [system_info.py:_probe_git():137] Probing git
+2024-12-31 22:20:46,968 DEBUG   HandlerThread:1756254 [system_info.py:_probe_git():145] Probing git done
+2024-12-31 22:20:46,968 DEBUG   HandlerThread:1756254 [system_info.py:probe():200] Probing system done
+2024-12-31 22:20:46,968 DEBUG   HandlerThread:1756254 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T03:20:46.962083', 'startedAt': '2025-01-01T03:20:46.569010', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.501, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.948, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.902, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 541.6344528198242}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-31 22:20:46,968 INFO    HandlerThread:1756254 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-31 22:20:46,968 INFO    HandlerThread:1756254 [system_monitor.py:probe():227] Publishing system info
+2024-12-31 22:20:46,968 DEBUG   HandlerThread:1756254 [system_info.py:_save_code():44] Saving code
+2024-12-31 22:20:46,970 DEBUG   HandlerThread:1756254 [system_info.py:_save_code():67] Saving code done
+2024-12-31 22:20:46,970 DEBUG   HandlerThread:1756254 [system_info.py:_save_patches():84] Saving git patches
+2024-12-31 22:20:47,032 DEBUG   HandlerThread:1756254 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-31 22:20:47,033 INFO    HandlerThread:1756254 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-31 22:20:47,036 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:20:47,036 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-31 22:20:47,036 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-31 22:20:47,036 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-31 22:20:47,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-31 22:20:47,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:20:47,277 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: python_packages
+2024-12-31 22:20:47,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:47,277 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:20:47,291 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/afq0b13k-wandb-metadata.json
+2024-12-31 22:20:47,308 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/2fskxp7n-diff.patch
+2024-12-31 22:20:47,321 INFO    HandlerThread:1756254 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/tb"
+  save: true
+}
+control {
+  relay_id: "5141a9007cc4479284808596a6f2d1f2"
+}
+_info {
+  stream_id: "9rj28b8z"
+}
+
+2024-12-31 22:20:47,321 DEBUG   HandlerThread:1756254 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-31 22:20:47,355 DEBUG   SenderThread:1756254 [sender.py:send():379] send: telemetry
+2024-12-31 22:20:47,373 DEBUG   SenderThread:1756254 [sender.py:send():379] send: tbrecord
+2024-12-31 22:20:47,374 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:20:47,374 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0 with policy live
+2024-12-31 22:20:47,423 INFO    wandb-upload_1:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/5v1kig1v-code/FISH/eval_robot.py
+2024-12-31 22:20:47,612 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/1c45muy0-events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/requirements.txt
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code/FISH/eval_robot.py
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-metadata.json
+2024-12-31 22:20:47,850 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/diff.patch
+2024-12-31 22:20:47,851 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code
+2024-12-31 22:20:47,851 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code/FISH
+2024-12-31 22:20:48,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:49,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:49,849 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:20:50,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:51,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:52,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:20:52,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:53,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:54,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:55,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:56,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:57,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:20:57,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:58,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:20:59,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:00,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:01,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:02,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:21:02,277 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:21:02,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:02,391 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:03,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:04,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:05,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:06,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:07,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:08,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:08,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:09,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:10,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:11,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:12,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:13,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:13,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:14,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:15,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:16,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:17,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:21:17,277 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:21:17,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:18,282 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:18,347 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:18,857 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml
+2024-12-31 22:21:19,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:20,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:21,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:22,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:23,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:24,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:24,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:25,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:26,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:27,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:28,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:29,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:29,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:30,284 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:31,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:31,316 DEBUG   SenderThread:1756254 [sender.py:send():379] send: config
+2024-12-31 22:21:32,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:21:32,277 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:21:32,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:33,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:33,861 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:34,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:34,414 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:35,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:36,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:37,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:37,862 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:38,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:39,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:39,849 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:39,863 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:40,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:41,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:42,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:43,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:44,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:44,867 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:45,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:45,865 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:46,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:46,952 DEBUG   SystemMonitor:1756254 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-31 22:21:46,952 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:21:47,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:21:47,278 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:21:47,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:48,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:49,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:49,866 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:50,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:50,635 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:50,866 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml
+2024-12-31 22:21:51,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:51,867 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:52,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:53,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:54,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:55,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:55,868 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:56,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:56,469 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:21:57,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:57,868 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:21:58,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:59,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:21:59,869 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:00,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:01,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:01,869 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:01,897 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:02,277 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:22:02,278 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:22:02,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:03,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:04,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:05,288 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:06,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:07,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:07,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:08,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:09,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:10,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:11,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:12,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:12,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:13,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:14,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:15,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:15,576 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:22:15,576 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4 with policy now
+2024-12-31 22:22:15,576 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:22:15,577 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:22:15,577 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:22:15,577 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval
+2024-12-31 22:22:15,874 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos
+2024-12-31 22:22:15,888 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/b0zjmynn-media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4
+2024-12-31 22:22:16,193 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/pbxrlqm2-media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4
+2024-12-31 22:22:16,290 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:16,953 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:22:17,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:22:17,278 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:22:17,325 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:17,433 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:17,875 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:18,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:19,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:20,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:21,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:21,877 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:22,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:23,283 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:23,291 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:23,878 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:24,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:24,878 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:22:25,138 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/zrevcriz-events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:22:25,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:25,878 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:26,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:27,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:27,879 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:28,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:29,007 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:29,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:29,880 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:30,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:31,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:32,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:22:32,278 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:22:32,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:33,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:34,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:34,386 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:35,293 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:36,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:37,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:38,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:39,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:40,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:40,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:41,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:42,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:43,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:44,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:45,295 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:46,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:46,295 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:46,953 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:22:47,279 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:22:47,280 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:22:47,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:48,295 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:49,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:50,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:51,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:51,430 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:52,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:53,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:54,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:55,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:55,888 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:56,078 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:22:56,078 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4 with policy now
+2024-12-31 22:22:56,083 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:22:56,084 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:22:56,084 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:22:56,085 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:22:56,296 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:56,341 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/1kh2rpcb-media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4
+2024-12-31 22:22:56,888 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:22:56,888 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4
+2024-12-31 22:22:56,889 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval
+2024-12-31 22:22:57,093 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:22:57,190 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/xa3bjmsq-media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4
+2024-12-31 22:22:57,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:57,889 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:58,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:22:58,889 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:22:59,305 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:00,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:01,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:01,890 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:02,280 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:23:02,281 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:23:02,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:02,381 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:03,299 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:03,891 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:04,300 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:05,300 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:05,891 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:23:06,300 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:06,754 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:23:06,755 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:23:06,756 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:23:06,892 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:23:07,300 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:07,892 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:08,286 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:08,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:09,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:10,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:11,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:11,894 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:12,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:13,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:13,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:14,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:15,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:16,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:16,954 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:23:17,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:23:17,279 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:23:17,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:18,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:18,386 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:19,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:20,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:21,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:22,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:23,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:24,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:24,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:25,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:26,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:27,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:28,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:29,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:30,289 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:30,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:31,303 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:32,278 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:23:32,279 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:23:32,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:33,304 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:34,304 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:35,304 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:35,436 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:36,304 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:37,307 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:37,658 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:23:37,659 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4 with policy now
+2024-12-31 22:23:37,662 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:23:37,663 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:23:37,663 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:23:37,663 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:23:37,902 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:37,902 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:23:37,902 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4
+2024-12-31 22:23:37,903 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval
+2024-12-31 22:23:37,929 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/ehm68xim-media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4
+2024-12-31 22:23:38,200 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/hgcfn8rn-media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4
+2024-12-31 22:23:38,308 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:38,902 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:39,309 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:39,903 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:40,308 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:40,692 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:40,903 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:41,312 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:42,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:43,312 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:43,904 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:44,310 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:45,310 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:45,905 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:45,908 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:46,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:46,586 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:23:46,586 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:23:46,586 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:23:46,905 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:23:46,905 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:23:46,954 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:23:47,168 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/lwhxb3fm-events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:23:47,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:23:47,287 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:23:47,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:47,905 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:48,318 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:49,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:50,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:51,259 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:51,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:51,907 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:23:52,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:53,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:54,315 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:55,315 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:56,301 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:23:56,315 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:57,315 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:58,315 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:59,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:23:59,910 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:00,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:01,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:01,834 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:02,281 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:24:02,281 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:24:02,325 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:03,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:03,911 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:04,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:05,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:06,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:07,021 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:07,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:08,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:09,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:10,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:11,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:12,302 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:12,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:13,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:14,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:15,319 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:16,317 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:16,955 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:24:17,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:24:17,286 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:24:17,323 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:17,387 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:17,916 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:18,325 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:18,329 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:24:18,329 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4 with policy now
+2024-12-31 22:24:18,377 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:24:18,378 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:24:18,378 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:24:18,378 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:24:18,595 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/95t011jx-media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4
+2024-12-31 22:24:18,917 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:24:18,917 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4
+2024-12-31 22:24:18,917 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval
+2024-12-31 22:24:19,192 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/aai7feos-media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4
+2024-12-31 22:24:19,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:19,917 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:20,336 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:20,917 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:21,325 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:22,321 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:23,314 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:23,332 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:23,918 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:24,328 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:25,335 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:25,919 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:26,332 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:27,346 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:28,335 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:29,312 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:29,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:30,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:30,921 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:24:31,338 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:31,642 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:24:31,642 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:24:31,642 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:24:31,921 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:31,921 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:24:32,292 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:24:32,293 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:24:32,346 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:33,338 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:33,922 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:34,350 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:34,417 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:35,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:35,923 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:24:36,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:37,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:38,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:39,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:40,308 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:40,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:41,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:42,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:43,337 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:44,338 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:45,308 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:45,338 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:46,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:46,955 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:24:47,285 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:24:47,286 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:24:47,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:48,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:49,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:50,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:50,363 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:51,344 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:52,347 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:53,347 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:54,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:55,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:56,309 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:24:56,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:57,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:58,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:24:59,349 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:00,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:01,348 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:01,781 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:02,294 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:25:02,294 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:25:02,361 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:02,996 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:25:02,996 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4 with policy now
+2024-12-31 22:25:02,999 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:25:03,007 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:25:03,007 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:03,008 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:03,272 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/r4s241db-media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4
+2024-12-31 22:25:03,356 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:03,932 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:03,933 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:03,933 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4
+2024-12-31 22:25:03,933 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval
+2024-12-31 22:25:04,233 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/yculme8b-media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4
+2024-12-31 22:25:04,369 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:05,364 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:06,379 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:07,312 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:07,372 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:07,934 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:08,369 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:09,469 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:10,473 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:11,472 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:11,935 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:12,316 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:12,482 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:13,479 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:14,479 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:15,484 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:15,695 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:25:15,695 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:15,696 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:15,937 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:15,937 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:25:16,185 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/ldqqyaau-events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:25:16,481 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:16,567 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:25:16,568 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:25:16,743 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:25:16,938 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:25:16,938 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media
+2024-12-31 22:25:16,938 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table
+2024-12-31 22:25:16,938 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval
+2024-12-31 22:25:17,121 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/4z7x5lq3-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:25:17,121 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:25:17,130 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:25:17,175 INFO    wandb-upload_0:1756254 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp78cj2gg3
+2024-12-31 22:25:17,287 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:25:17,471 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:17,758 INFO    SenderThread:1756254 [sender.py:send_request_log_artifact():1525] logged artifact run-9rj28b8z-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE1OTU5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MjczOQ==', 'latestArtifact': None}}
+2024-12-31 22:25:17,759 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:25:17,759 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2024-12-31 22:25:17,759 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:25:17,759 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,759 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:17,760 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,760 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:25:17,760 DEBUG   SenderThread:1756254 [sender.py:send():379] send: telemetry
+2024-12-31 22:25:17,760 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,760 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,761 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,761 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,761 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,763 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,763 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,763 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,763 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,764 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,764 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,764 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,764 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,764 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,765 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,765 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,765 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,765 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,765 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,766 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,766 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:17,766 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:17,766 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:25:17,938 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:17,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:17,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:25:17,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json
+2024-12-31 22:25:17,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval
+2024-12-31 22:25:18,012 INFO    wandb-upload_1:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/kspg8hf6-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:25:18,178 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/qtrdskjr-media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json
+2024-12-31 22:25:18,273 INFO    wandb-upload_2:1756254 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5_8snr6f
+2024-12-31 22:25:18,471 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:18,774 INFO    SenderThread:1756254 [sender.py:send_request_log_artifact():1525] logged artifact run-9rj28b8z-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE1OTYwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3Mjc0MA==', 'latestArtifact': None}}
+2024-12-31 22:25:18,774 DEBUG   SenderThread:1756254 [sender.py:send():379] send: files
+2024-12-31 22:25:18,774 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json with policy now
+2024-12-31 22:25:18,774 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:25:18,774 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:18,775 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:18,775 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:25:18,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:19,027 INFO    wandb-upload_1:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/in3hzrf4-media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json
+2024-12-31 22:25:19,257 DEBUG   SenderThread:1756254 [sender.py:send():379] send: exit
+2024-12-31 22:25:19,258 INFO    SenderThread:1756254 [sender.py:send_exit():586] handling exit code: 0
+2024-12-31 22:25:19,258 INFO    SenderThread:1756254 [sender.py:send_exit():588] handling runtime: 272
+2024-12-31 22:25:19,258 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:19,258 INFO    SenderThread:1756254 [sender.py:send_exit():594] send defer
+2024-12-31 22:25:19,258 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:19,258 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-31 22:25:19,258 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:19,259 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-31 22:25:19,259 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 1
+2024-12-31 22:25:19,259 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:19,259 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-31 22:25:19,259 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:19,259 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-31 22:25:19,259 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 2
+2024-12-31 22:25:19,259 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:19,259 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-31 22:25:19,259 INFO    HandlerThread:1756254 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-31 22:25:19,259 DEBUG   SystemMonitor:1756254 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-31 22:25:19,259 DEBUG   SystemMonitor:1756254 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-31 22:25:19,260 INFO    HandlerThread:1756254 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-31 22:25:19,260 INFO    HandlerThread:1756254 [interfaces.py:finish():200] Joined disk monitor
+2024-12-31 22:25:19,312 INFO    HandlerThread:1756254 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-31 22:25:19,312 INFO    HandlerThread:1756254 [interfaces.py:finish():200] Joined memory monitor
+2024-12-31 22:25:19,312 INFO    HandlerThread:1756254 [interfaces.py:finish():200] Joined network monitor
+2024-12-31 22:25:19,313 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:19,313 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-31 22:25:19,313 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 3
+2024-12-31 22:25:19,313 DEBUG   SenderThread:1756254 [sender.py:send():379] send: stats
+2024-12-31 22:25:19,313 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:19,313 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-31 22:25:19,313 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:19,313 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-31 22:25:19,313 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 4
+2024-12-31 22:25:19,313 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:19,313 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-31 22:25:19,939 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:19,940 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:21,940 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:25,704 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:25,705 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:25,705 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-31 22:25:25,705 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:25,706 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-31 22:25:25,706 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 5
+2024-12-31 22:25:25,706 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:25,706 DEBUG   SenderThread:1756254 [sender.py:send():379] send: history
+2024-12-31 22:25:25,706 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:25:25,706 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:25,706 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-31 22:25:25,707 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:25,707 DEBUG   SenderThread:1756254 [sender.py:send():379] send: summary
+2024-12-31 22:25:25,711 INFO    SenderThread:1756254 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:25:25,711 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:25,711 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-31 22:25:25,711 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 6
+2024-12-31 22:25:25,711 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:25,711 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-31 22:25:25,711 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:25,711 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-31 22:25:25,715 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:25:25,826 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 7
+2024-12-31 22:25:25,826 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:25,826 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-31 22:25:25,826 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:25,826 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-31 22:25:25,942 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:25,942 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml
+2024-12-31 22:25:26,259 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:28,222 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 8
+2024-12-31 22:25:28,222 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:28,222 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:28,222 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-31 22:25:28,223 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:28,223 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-31 22:25:28,223 INFO    SenderThread:1756254 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-31 22:25:28,227 INFO    SenderThread:1756254 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-31 22:25:28,229 INFO    SenderThread:1756254 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-31 22:25:28,248 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 9
+2024-12-31 22:25:28,248 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:28,248 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-31 22:25:28,249 DEBUG   SenderThread:1756254 [sender.py:send():379] send: artifact
+2024-12-31 22:25:28,259 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:28,635 INFO    wandb-upload_0:1756254 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmput398yfj
+2024-12-31 22:25:28,636 INFO    wandb-upload_1:1756254 [upload_job.py:push():85] Skipped uploading /tmp/tmp_y59j39j/wandb-job.json
+2024-12-31 22:25:28,760 INFO    wandb-upload_2:1756254 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpwld62_lp
+2024-12-31 22:25:28,943 INFO    Thread-12 :1756254 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:29,418 INFO    SenderThread:1756254 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE1OTY3NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE1NjI2NQ==', 'versionIndex': 395}}}
+2024-12-31 22:25:29,418 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:29,418 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-31 22:25:29,418 INFO    SenderThread:1756254 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml config.yaml
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json wandb-summary.json
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/requirements.txt requirements.txt
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-metadata.json wandb-metadata.json
+2024-12-31 22:25:29,944 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0 events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:25:29,945 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log output.log
+2024-12-31 22:25:29,948 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/diff.patch diff.patch
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4 media/videos/eval/1_eval_1_d1adf501ad4a659c079d.mp4
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4 media/videos/eval/3_eval_3_3ad55986d23b06cab972.mp4
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4 media/videos/eval/0_eval_0_4375a602e73826f9f16a.mp4
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4 media/videos/eval/2_eval_2_2496e43e4bbc06af2a2a.mp4
+2024-12-31 22:25:29,950 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4 media/videos/eval/4_eval_4_209d1fd47e66e5a5c21c.mp4
+2024-12-31 22:25:29,952 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json media/table/eval/total_success_rate_ci_6_dd484121e9f12436bed3.table.json
+2024-12-31 22:25:29,952 INFO    SenderThread:1756254 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:25:29,953 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 10
+2024-12-31 22:25:29,953 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:29,953 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:29,954 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-31 22:25:29,954 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:29,954 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-31 22:25:29,954 INFO    SenderThread:1756254 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:25:30,135 INFO    wandb-upload_0:1756254 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/config.yaml
+2024-12-31 22:25:30,152 INFO    wandb-upload_2:1756254 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/requirements.txt
+2024-12-31 22:25:30,159 INFO    wandb-upload_1:1756254 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/wandb-summary.json
+2024-12-31 22:25:30,165 INFO    wandb-upload_4:1756254 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/files/output.log
+2024-12-31 22:25:30,259 INFO    wandb-upload_3:1756254 [upload_job.py:push():130] Uploaded file /tmp/tmp8wa6a5izwandb/q78ua6a5-events.out.tfevents.1735701647.leonmkim-ROG-Strix-G15CS-G15CS.1756086.0
+2024-12-31 22:25:30,260 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:30,260 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:30,460 INFO    Thread-11 (_thread_body):1756254 [sender.py:transition_state():614] send defer: 11
+2024-12-31 22:25:30,460 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:30,460 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-31 22:25:30,460 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:30,460 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-31 22:25:30,461 INFO    SenderThread:1756254 [file_pusher.py:join():175] waiting for file pusher
+2024-12-31 22:25:30,462 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 12
+2024-12-31 22:25:30,462 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:30,462 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-31 22:25:30,462 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:30,462 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-31 22:25:30,462 INFO    SenderThread:1756254 [file_stream.py:finish():601] file stream finish called
+2024-12-31 22:25:30,519 INFO    SenderThread:1756254 [file_stream.py:finish():605] file stream finish is done
+2024-12-31 22:25:30,519 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 13
+2024-12-31 22:25:30,520 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:30,520 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-31 22:25:30,520 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:30,520 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-31 22:25:30,520 INFO    SenderThread:1756254 [sender.py:transition_state():614] send defer: 14
+2024-12-31 22:25:30,520 DEBUG   SenderThread:1756254 [sender.py:send():379] send: final
+2024-12-31 22:25:30,520 DEBUG   SenderThread:1756254 [sender.py:send():379] send: footer
+2024-12-31 22:25:30,520 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:25:30,520 INFO    HandlerThread:1756254 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-31 22:25:30,521 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:25:30,521 INFO    SenderThread:1756254 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-31 22:25:30,521 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:30,521 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:30,522 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: server_info
+2024-12-31 22:25:30,522 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:25:30,522 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-31 22:25:30,522 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-31 22:25:30,522 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:25:30,523 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: server_info
+2024-12-31 22:25:30,575 DEBUG   SenderThread:1756254 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:25:30,575 INFO    MainThread:1756254 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-31 22:25:30,576 INFO    MainThread:1756254 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-31 22:25:30,576 INFO    MainThread:1756254 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-31 22:25:30,576 DEBUG   HandlerThread:1756254 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-31 22:25:30,576 INFO    HandlerThread:1756254 [handler.py:finish():882] shutting down handler
+2024-12-31 22:25:31,523 INFO    WriterThread:1756254 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/run-9rj28b8z.wandb
+2024-12-31 22:25:31,575 INFO    SenderThread:1756254 [sender.py:finish():1615] shutting down sender
+2024-12-31 22:25:31,575 INFO    SenderThread:1756254 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:25:31,575 INFO    SenderThread:1756254 [file_pusher.py:join():175] waiting for file pusher
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/logs/debug.log b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..d31bf16dc6ebd29de616c8c5e3784ee1996d89a5
--- /dev/null
+++ b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/logs/debug.log
@@ -0,0 +1,31 @@
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Configure stats pid to 1756086
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/settings
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/logs/debug.log
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/logs/debug-internal.log
+2024-12-31 22:20:46,575 INFO    MainThread:1756086 [wandb_init.py:init():569] calling init triggers
+2024-12-31 22:20:46,576 INFO    MainThread:1756086 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z'}
+2024-12-31 22:20:46,576 INFO    MainThread:1756086 [wandb_init.py:init():619] starting backend
+2024-12-31 22:20:46,576 INFO    MainThread:1756086 [wandb_init.py:init():623] setting up manager
+2024-12-31 22:20:46,581 INFO    MainThread:1756086 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-31 22:20:46,585 INFO    MainThread:1756086 [wandb_init.py:init():631] backend started and connected
+2024-12-31 22:20:46,599 INFO    MainThread:1756086 [wandb_init.py:init():720] updated telemetry
+2024-12-31 22:20:46,605 INFO    MainThread:1756086 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-31 22:20:46,867 INFO    MainThread:1756086 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-31 22:20:46,940 INFO    MainThread:1756086 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-31 22:20:46,940 INFO    MainThread:1756086 [wandb_init.py:init():804] starting run threads in backend
+2024-12-31 22:20:47,277 INFO    MainThread:1756086 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-31 22:20:47,278 INFO    MainThread:1756086 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-31 22:20:47,278 INFO    MainThread:1756086 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-31 22:20:47,278 INFO    MainThread:1756086 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-31 22:20:47,280 INFO    MainThread:1756086 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-31 22:20:47,281 INFO    MainThread:1756086 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/9rj28b8z/tb, True
+2024-12-31 22:21:31,315 INFO    MainThread:1756086 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-31 22:25:31,577 WARNING MsgRouterThr:1756086 [router.py:message_loop():77] message_loop has been closed
diff --git a/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/run-9rj28b8z.wandb b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/run-9rj28b8z.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..02a3909bd40d8dba98dc838bd07d13927ef6abc0
Binary files /dev/null and b/9rj28b8z/wandb/run-20241231_222046-9rj28b8z/run-9rj28b8z.wandb differ
diff --git a/gwnrzhmx/.hydra/config.yaml b/gwnrzhmx/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe
--- /dev/null
+++ b/gwnrzhmx/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1007_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/gwnrzhmx/.hydra/hydra.yaml b/gwnrzhmx/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fd6318ae83a7437c8b24e74a293d01ce2fe8c87e
--- /dev/null
+++ b/gwnrzhmx/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/gwnrzhmx/.hydra/overrides.yaml b/gwnrzhmx/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/gwnrzhmx/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/gwnrzhmx/episode_rosbags/aligned_depth_to_color_K.npy b/gwnrzhmx/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/gwnrzhmx/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/gwnrzhmx/episode_rosbags/cam_tf_world.npy b/gwnrzhmx/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/gwnrzhmx/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/gwnrzhmx/episode_rosbags/color_K.npy b/gwnrzhmx/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/gwnrzhmx/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/gwnrzhmx/episode_rosbags/depth_K.npy b/gwnrzhmx/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/gwnrzhmx/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/gwnrzhmx/eval_robot.log b/gwnrzhmx/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..e75b8037a0c380da2d6a4d6f83fcb0b455117763
--- /dev/null
+++ b/gwnrzhmx/eval_robot.log
@@ -0,0 +1,12 @@
+[2024-12-31 22:14:56,081][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-31 22:14:56,084][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/gwnrzhmx/eval_video/0_eval.mp4 b/gwnrzhmx/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..2b316fcffdf854924907f3401a9fdf64996c3095
--- /dev/null
+++ b/gwnrzhmx/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c17a3e9a7bb8fc87a935a218fd8132e73eba1da41b97e1cd42c23ef5c1e26cf9
+size 1139182
diff --git a/gwnrzhmx/eval_video/1_eval.mp4 b/gwnrzhmx/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..88443191ad26560f5dcced4d4c17533f91d94bae
--- /dev/null
+++ b/gwnrzhmx/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db2109c40f910f358754ba1da0965361ab3b4a6adb800a0207d1a7d0bdfd0d2b
+size 1217463
diff --git a/gwnrzhmx/eval_video/2_eval.mp4 b/gwnrzhmx/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..311c870bd14268ef235c44851561c05fd41e4754
--- /dev/null
+++ b/gwnrzhmx/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29c79964510fc68204d881dae01ac65d1e42ad65d896e0c56cd6bc77f781a4c0
+size 1214674
diff --git a/gwnrzhmx/eval_video/3_eval.mp4 b/gwnrzhmx/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..6b867ad7e408f8154de6b0937a0aa9e3b370d1ab
--- /dev/null
+++ b/gwnrzhmx/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d71847ccf69c2b06c3e663657321a935a91869ddcf855e7e53d2eea24d3f63f
+size 1176392
diff --git a/gwnrzhmx/eval_video/4_eval.mp4 b/gwnrzhmx/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..36927230fc92514f533a982f8443eb49c3957b70
--- /dev/null
+++ b/gwnrzhmx/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a9cd71b9b9e63fcffc667c82fce584257e4da0cf58a858b45ec50af0c8d6fc4
+size 1223048
diff --git a/gwnrzhmx/tb/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0 b/gwnrzhmx/tb/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
new file mode 100644
index 0000000000000000000000000000000000000000..5cf75c29f6e27ccb5e85ca3ce3ffac38128bffda
--- /dev/null
+++ b/gwnrzhmx/tb/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16890976787d0416f28a7fcadee56f063410d7c2458e9a47b6a3287fdda9e0b0
+size 6539
diff --git a/gwnrzhmx/wandb/debug-internal.log b/gwnrzhmx/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..12bd616b6c898ca0ed2598a9c890991267a2e2cd
--- /dev/null
+++ b/gwnrzhmx/wandb/debug-internal.log
@@ -0,0 +1,748 @@
+2024-12-31 22:15:02,172 INFO    StreamThr :1754882 [internal.py:wandb_internal():85] W&B internal server running at pid: 1754882, started at: 2024-12-31 22:15:02.171894
+2024-12-31 22:15:02,173 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status
+2024-12-31 22:15:02,185 INFO    WriterThread:1754882 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/run-gwnrzhmx.wandb
+2024-12-31 22:15:02,185 DEBUG   SenderThread:1754882 [sender.py:send():379] send: header
+2024-12-31 22:15:02,195 DEBUG   SenderThread:1754882 [sender.py:send():379] send: run
+2024-12-31 22:15:02,394 INFO    SenderThread:1754882 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files
+2024-12-31 22:15:02,394 INFO    SenderThread:1754882 [sender.py:_start_run_threads():1188] run started: gwnrzhmx with start time 1735701302.172689
+2024-12-31 22:15:02,410 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: check_version
+2024-12-31 22:15:02,410 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: check_version
+2024-12-31 22:15:02,481 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: run_start
+2024-12-31 22:15:02,486 DEBUG   HandlerThread:1754882 [system_info.py:__init__():26] System info init
+2024-12-31 22:15:02,486 DEBUG   HandlerThread:1754882 [system_info.py:__init__():41] System info init done
+2024-12-31 22:15:02,486 INFO    HandlerThread:1754882 [system_monitor.py:start():194] Starting system monitor
+2024-12-31 22:15:02,486 INFO    SystemMonitor:1754882 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-31 22:15:02,486 INFO    HandlerThread:1754882 [system_monitor.py:probe():214] Collecting system info
+2024-12-31 22:15:02,486 INFO    SystemMonitor:1754882 [interfaces.py:start():188] Started cpu monitoring
+2024-12-31 22:15:02,487 INFO    SystemMonitor:1754882 [interfaces.py:start():188] Started disk monitoring
+2024-12-31 22:15:02,488 INFO    SystemMonitor:1754882 [interfaces.py:start():188] Started gpu monitoring
+2024-12-31 22:15:02,488 INFO    SystemMonitor:1754882 [interfaces.py:start():188] Started memory monitoring
+2024-12-31 22:15:02,488 INFO    SystemMonitor:1754882 [interfaces.py:start():188] Started network monitoring
+2024-12-31 22:15:02,500 DEBUG   HandlerThread:1754882 [system_info.py:probe():152] Probing system
+2024-12-31 22:15:02,501 DEBUG   HandlerThread:1754882 [system_info.py:_probe_git():137] Probing git
+2024-12-31 22:15:02,506 DEBUG   HandlerThread:1754882 [system_info.py:_probe_git():145] Probing git done
+2024-12-31 22:15:02,506 DEBUG   HandlerThread:1754882 [system_info.py:probe():200] Probing system done
+2024-12-31 22:15:02,506 DEBUG   HandlerThread:1754882 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T03:15:02.500360', 'startedAt': '2025-01-01T03:15:02.164240', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.02675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.012, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.046, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4495.794, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 535.0275993347168}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-31 22:15:02,506 INFO    HandlerThread:1754882 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-31 22:15:02,506 INFO    HandlerThread:1754882 [system_monitor.py:probe():227] Publishing system info
+2024-12-31 22:15:02,506 DEBUG   HandlerThread:1754882 [system_info.py:_save_code():44] Saving code
+2024-12-31 22:15:02,508 DEBUG   HandlerThread:1754882 [system_info.py:_save_code():67] Saving code done
+2024-12-31 22:15:02,508 DEBUG   HandlerThread:1754882 [system_info.py:_save_patches():84] Saving git patches
+2024-12-31 22:15:02,570 DEBUG   HandlerThread:1754882 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-31 22:15:02,570 INFO    HandlerThread:1754882 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-31 22:15:02,573 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:15:02,573 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-31 22:15:02,574 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-31 22:15:02,574 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-31 22:15:02,834 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-31 22:15:02,834 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: python_packages
+2024-12-31 22:15:02,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:02,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:15:02,836 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:15:02,842 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/6oatoedz-wandb-metadata.json
+2024-12-31 22:15:02,854 INFO    wandb-upload_2:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/wizkghbd-diff.patch
+2024-12-31 22:15:02,877 INFO    HandlerThread:1754882 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/tb"
+  save: true
+}
+control {
+  relay_id: "25a480f1578d4ed69d39e4d6a73bba29"
+}
+_info {
+  stream_id: "gwnrzhmx"
+}
+
+2024-12-31 22:15:02,877 DEBUG   HandlerThread:1754882 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-31 22:15:02,902 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/adxeg02d-code/FISH/eval_robot.py
+2024-12-31 22:15:02,917 DEBUG   SenderThread:1754882 [sender.py:send():379] send: telemetry
+2024-12-31 22:15:02,929 DEBUG   SenderThread:1754882 [sender.py:send():379] send: tbrecord
+2024-12-31 22:15:02,929 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:15:02,930 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0 with policy live
+2024-12-31 22:15:03,131 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/swkfhmz1-events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:15:03,395 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code/FISH/eval_robot.py
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-metadata.json
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/diff.patch
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/requirements.txt
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code/FISH
+2024-12-31 22:15:03,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:04,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:05,395 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:05,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:06,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:07,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:07,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:08,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:09,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:10,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:11,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:12,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:12,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:13,401 DEBUG   SenderThread:1754882 [sender.py:send():379] send: config
+2024-12-31 22:15:13,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:14,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:15,398 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:15,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:16,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:17,398 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:17,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:15:17,835 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:15:17,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:17,891 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:18,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:19,399 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:19,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:20,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:21,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:22,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:23,400 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:23,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:23,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:24,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:25,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:26,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:27,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:28,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:29,097 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:29,402 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:29,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:30,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:31,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:32,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:15:32,835 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:15:32,877 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:33,403 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:33,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:34,733 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:34,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:35,403 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:35,403 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml
+2024-12-31 22:15:35,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:36,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:37,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:38,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:39,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:40,797 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:40,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:41,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:42,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:43,405 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:43,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:44,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:45,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:45,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:46,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:47,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:15:47,835 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:15:47,877 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:48,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:49,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:50,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:50,973 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:51,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:52,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:53,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:54,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:55,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:56,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:56,845 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:57,843 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:58,726 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:15:58,726 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4 with policy now
+2024-12-31 22:15:58,729 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:15:58,730 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:15:58,730 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:15:58,731 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:15:58,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:59,012 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/0uoo3djq-media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4
+2024-12-31 22:15:59,410 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:59,410 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4
+2024-12-31 22:15:59,411 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:15:59,411 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval
+2024-12-31 22:15:59,411 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos
+2024-12-31 22:15:59,411 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media
+2024-12-31 22:15:59,692 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/0wj9cy4r-media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4
+2024-12-31 22:15:59,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:00,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:01,411 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:01,845 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:02,139 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:02,489 DEBUG   SystemMonitor:1754882 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-31 22:16:02,489 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:16:02,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:16:02,836 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:16:02,845 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:03,412 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:03,845 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:04,849 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:05,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:06,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:07,413 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:07,413 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:16:07,665 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/1rsoj03n-events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:16:07,829 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:07,877 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:08,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:09,414 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:09,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:10,845 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:11,850 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:12,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:12,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:13,415 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:13,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:14,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:15,416 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:15,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:16,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:17,416 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:17,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:16:17,836 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:16:17,877 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:17,960 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:18,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:19,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:20,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:21,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:22,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:23,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:23,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:24,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:25,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:26,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:27,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:28,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:28,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:29,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:30,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:31,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:32,490 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:16:32,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:16:32,836 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:16:32,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:33,421 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:33,856 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:33,987 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:34,856 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:35,422 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:35,856 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:36,856 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:37,856 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:38,857 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:39,489 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:39,737 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:16:39,737 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_db2109c40f910f358754.mp4 with policy now
+2024-12-31 22:16:39,738 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:16:39,738 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:16:39,738 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:16:39,739 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:16:39,857 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:40,009 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/urlv7djh-media/videos/eval/1_eval_1_db2109c40f910f358754.mp4
+2024-12-31 22:16:40,424 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:16:40,424 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/1_eval_1_db2109c40f910f358754.mp4
+2024-12-31 22:16:40,424 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval
+2024-12-31 22:16:40,733 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/52bxirtc-media/videos/eval/1_eval_1_db2109c40f910f358754.mp4
+2024-12-31 22:16:40,861 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:41,424 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:41,858 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:42,860 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:43,859 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:44,859 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:45,005 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:45,425 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:45,859 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:46,861 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:47,274 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:16:47,274 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:16:47,274 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:16:47,426 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:47,426 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:16:47,426 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:16:47,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:16:47,840 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:16:47,863 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:48,865 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:49,427 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:49,864 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:50,033 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:50,864 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:51,861 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:52,862 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:53,428 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:53,862 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:54,868 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:55,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:55,867 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:56,867 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:57,867 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:58,867 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:59,868 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:00,868 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:01,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:01,870 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:02,490 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:17:02,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:17:02,838 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:17:02,881 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:03,873 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:04,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:05,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:06,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:06,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:07,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:08,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:09,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:10,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:11,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:12,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:12,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:13,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:14,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:15,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:16,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:17,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:17,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:17:17,845 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:17:17,885 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:18,879 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:19,882 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:20,128 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:17:20,128 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4 with policy now
+2024-12-31 22:17:20,138 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:17:20,139 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:17:20,139 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:17:20,139 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:17:20,391 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/kzf77wj5-media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4
+2024-12-31 22:17:20,437 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:17:20,437 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4
+2024-12-31 22:17:20,437 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval
+2024-12-31 22:17:20,705 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/mqgrvrh4-media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4
+2024-12-31 22:17:20,884 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:21,437 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:21,887 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:22,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:22,892 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:23,888 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:24,890 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:25,439 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:25,887 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:26,892 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:27,440 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:27,850 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:27,889 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:28,890 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:29,896 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:30,893 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:31,894 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:32,491 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:17:32,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:17:32,853 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:17:32,901 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:32,933 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:33,889 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:34,893 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:35,113 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:17:35,113 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:17:35,114 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:17:35,442 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:17:35,442 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:17:35,727 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/4uw8m5po-events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:17:35,894 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:36,901 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:37,443 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:37,901 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:38,549 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:38,898 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:39,443 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:39,906 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:40,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:41,444 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:41,901 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:42,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:43,445 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:43,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:43,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:44,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:45,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:46,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:47,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:17:47,842 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:17:47,903 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:48,903 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:48,981 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:49,903 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:50,903 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:51,903 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:52,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:53,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:54,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:54,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:55,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:56,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:57,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:58,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:59,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:59,905 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:00,905 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:01,905 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:02,491 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:18:02,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:18:02,842 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:18:02,905 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:03,905 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:04,908 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:05,002 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:05,908 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:06,911 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:07,698 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:18:07,698 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4 with policy now
+2024-12-31 22:18:07,704 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:18:07,704 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:18:07,705 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:18:07,707 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:18:07,916 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:07,982 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/1rq1l2ra-media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4
+2024-12-31 22:18:08,453 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:18:08,453 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4
+2024-12-31 22:18:08,453 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval
+2024-12-31 22:18:08,750 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/4sqoioh6-media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4
+2024-12-31 22:18:08,916 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:09,453 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:09,916 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:10,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:10,915 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:11,929 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:12,927 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:13,455 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:13,932 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:14,918 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:15,455 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:15,930 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:15,960 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:16,165 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:18:16,165 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:18:16,165 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:18:16,456 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:18:16,456 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:18:16,918 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:17,456 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:17,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:18:17,855 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:18:17,929 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:18,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:19,457 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:19,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:20,927 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:21,458 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:21,698 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:21,922 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:22,922 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:23,458 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:23,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:24,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:25,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:26,849 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:26,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:27,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:28,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:29,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:30,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:31,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:32,492 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:18:32,493 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:32,843 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:18:32,843 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:18:32,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:33,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:34,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:35,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:36,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:37,850 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:37,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:38,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:39,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:40,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:41,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:42,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:43,850 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:43,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:44,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:45,466 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:45,926 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:46,926 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:47,467 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:47,843 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:18:47,843 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:18:47,926 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:48,906 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:48,926 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:49,949 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:50,213 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:18:50,213 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:18:50,213 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4 with policy now
+2024-12-31 22:18:50,214 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:18:50,214 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:18:50,214 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:18:50,468 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:18:50,468 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4
+2024-12-31 22:18:50,468 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval
+2024-12-31 22:18:50,472 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/8k6l9wqp-media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4
+2024-12-31 22:18:50,732 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/zy1m3qkj-media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4
+2024-12-31 22:18:50,942 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:51,468 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:51,934 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:52,942 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:53,942 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:54,789 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:54,935 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:55,469 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:55,940 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:56,944 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:57,470 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:57,959 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:59,009 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:59,216 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:18:59,216 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:18:59,217 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:18:59,471 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:18:59,471 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:18:59,714 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/xu2xzo59-events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:18:59,892 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:18:59,892 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:18:59,949 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:00,073 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:19:00,347 INFO    wandb-upload_0:1754882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp79jlbk65
+2024-12-31 22:19:00,469 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:19:00,471 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:19:00,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json
+2024-12-31 22:19:00,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media
+2024-12-31 22:19:00,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table
+2024-12-31 22:19:00,473 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval
+2024-12-31 22:19:00,477 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:19:00,740 INFO    SenderThread:1754882 [sender.py:send_request_log_artifact():1525] logged artifact run-gwnrzhmx-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE1NjEwNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MjI1Ng==', 'latestArtifact': None}}
+2024-12-31 22:19:00,740 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:19:00,740 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now
+2024-12-31 22:19:00,742 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:19:00,742 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,742 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:19:00,743 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,743 DEBUG   SenderThread:1754882 [sender.py:send():379] send: telemetry
+2024-12-31 22:19:00,743 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,743 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,743 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,744 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,744 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,744 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,744 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,745 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,745 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,745 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,745 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,746 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,746 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,746 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,746 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,746 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,747 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,747 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,747 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,747 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,748 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,748 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,748 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:19:00,946 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:00,949 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/2nfsdqaf-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json
+2024-12-31 22:19:01,100 INFO    wandb-upload_0:1754882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3mt8271m
+2024-12-31 22:19:01,197 INFO    wandb-upload_2:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/9olnewwt-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json
+2024-12-31 22:19:01,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:19:01,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:19:01,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json
+2024-12-31 22:19:01,473 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval
+2024-12-31 22:19:01,529 INFO    SenderThread:1754882 [sender.py:send_request_log_artifact():1525] logged artifact run-gwnrzhmx-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE1NjExMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MjI1Nw==', 'latestArtifact': None}}
+2024-12-31 22:19:01,529 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:19:01,529 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json with policy now
+2024-12-31 22:19:01,531 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:19:01,531 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:01,532 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:01,752 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/zc6s33tb-media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json
+2024-12-31 22:19:01,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:02,000 INFO    wandb-upload_2:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/x16wgeuo-media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json
+2024-12-31 22:19:02,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:19:02,492 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:19:02,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:19:02,844 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:19:02,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:03,473 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:19:03,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:04,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:05,943 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:19:05,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:06,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:07,947 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:08,946 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:09,948 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:10,946 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:11,173 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:19:11,221 DEBUG   SenderThread:1754882 [sender.py:send():379] send: exit
+2024-12-31 22:19:11,221 INFO    SenderThread:1754882 [sender.py:send_exit():586] handling exit code: 0
+2024-12-31 22:19:11,221 INFO    SenderThread:1754882 [sender.py:send_exit():588] handling runtime: 248
+2024-12-31 22:19:11,223 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:11,223 INFO    SenderThread:1754882 [sender.py:send_exit():594] send defer
+2024-12-31 22:19:11,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:11,223 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-31 22:19:11,223 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:11,223 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-31 22:19:11,223 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 1
+2024-12-31 22:19:11,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:11,223 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-31 22:19:11,223 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:11,223 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-31 22:19:11,224 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 2
+2024-12-31 22:19:11,224 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:11,224 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-31 22:19:11,224 INFO    HandlerThread:1754882 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-31 22:19:11,224 DEBUG   SystemMonitor:1754882 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-31 22:19:11,224 DEBUG   SystemMonitor:1754882 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-31 22:19:11,225 INFO    HandlerThread:1754882 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-31 22:19:11,225 INFO    HandlerThread:1754882 [interfaces.py:finish():200] Joined disk monitor
+2024-12-31 22:19:11,276 INFO    HandlerThread:1754882 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-31 22:19:11,276 INFO    HandlerThread:1754882 [interfaces.py:finish():200] Joined memory monitor
+2024-12-31 22:19:11,276 INFO    HandlerThread:1754882 [interfaces.py:finish():200] Joined network monitor
+2024-12-31 22:19:11,277 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:11,277 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-31 22:19:11,277 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 3
+2024-12-31 22:19:11,277 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:19:11,277 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:11,277 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-31 22:19:11,277 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:11,277 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-31 22:19:11,277 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 4
+2024-12-31 22:19:11,278 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:11,278 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-31 22:19:11,476 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:19:13,477 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:19:18,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:19:18,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:19:18,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-31 22:19:18,224 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:18,224 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-31 22:19:18,224 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 5
+2024-12-31 22:19:18,224 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:19:18,224 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:19:18,224 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:18,225 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:18,225 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-31 22:19:18,225 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:18,226 DEBUG   SenderThread:1754882 [sender.py:send():379] send: summary
+2024-12-31 22:19:18,229 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:18,229 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:18,230 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-31 22:19:18,230 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 6
+2024-12-31 22:19:18,230 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:18,230 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-31 22:19:18,230 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:18,230 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-31 22:19:18,233 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:19:18,352 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 7
+2024-12-31 22:19:18,352 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:18,352 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-31 22:19:18,352 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:18,352 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-31 22:19:18,479 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml
+2024-12-31 22:19:18,479 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:19:19,193 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 8
+2024-12-31 22:19:19,193 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:19,193 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-31 22:19:19,193 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:19,194 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-31 22:19:19,194 INFO    SenderThread:1754882 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-31 22:19:19,196 INFO    SenderThread:1754882 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-31 22:19:19,198 INFO    SenderThread:1754882 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-31 22:19:19,208 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 9
+2024-12-31 22:19:19,208 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:19,208 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-31 22:19:19,209 DEBUG   SenderThread:1754882 [sender.py:send():379] send: artifact
+2024-12-31 22:19:19,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:19:19,479 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:19:19,540 INFO    wandb-upload_0:1754882 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpio3ffp0z
+2024-12-31 22:19:19,540 INFO    wandb-upload_2:1754882 [upload_job.py:push():85] Skipped uploading /tmp/tmpcq73m_vk/wandb-job.json
+2024-12-31 22:19:19,648 INFO    wandb-upload_1:1754882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphqylxoc7
+2024-12-31 22:19:20,228 INFO    SenderThread:1754882 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE1NjI2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjIyOA==', 'versionIndex': 394}}}
+2024-12-31 22:19:20,228 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:20,228 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-31 22:19:20,228 INFO    SenderThread:1754882 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-31 22:19:20,480 INFO    SenderThread:1754882 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files
+2024-12-31 22:19:20,480 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml config.yaml
+2024-12-31 22:19:20,480 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json wandb-summary.json
+2024-12-31 22:19:20,480 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0 events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:19:20,481 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/requirements.txt requirements.txt
+2024-12-31 22:19:20,485 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-metadata.json wandb-metadata.json
+2024-12-31 22:19:20,485 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log output.log
+2024-12-31 22:19:20,486 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/diff.patch diff.patch
+2024-12-31 22:19:20,486 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-31 22:19:20,486 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/1_eval_1_db2109c40f910f358754.mp4 media/videos/eval/1_eval_1_db2109c40f910f358754.mp4
+2024-12-31 22:19:20,490 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4 media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4
+2024-12-31 22:19:20,491 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4 media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4
+2024-12-31 22:19:20,491 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4 media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4
+2024-12-31 22:19:20,491 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4 media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4
+2024-12-31 22:19:20,493 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json
+2024-12-31 22:19:20,493 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json
+2024-12-31 22:19:20,493 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 10
+2024-12-31 22:19:20,493 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:19:20,493 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:20,493 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-31 22:19:20,494 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:20,494 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-31 22:19:20,494 INFO    SenderThread:1754882 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:19:20,698 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml
+2024-12-31 22:19:20,721 INFO    wandb-upload_3:1754882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/requirements.txt
+2024-12-31 22:19:20,725 INFO    wandb-upload_4:1754882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:19:20,735 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/vavfk9xn-events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:19:20,739 INFO    wandb-upload_2:1754882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:19:20,939 INFO    Thread-11 (_thread_body):1754882 [sender.py:transition_state():614] send defer: 11
+2024-12-31 22:19:20,940 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:20,940 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-31 22:19:20,940 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:20,940 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-31 22:19:20,940 INFO    SenderThread:1754882 [file_pusher.py:join():175] waiting for file pusher
+2024-12-31 22:19:20,941 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 12
+2024-12-31 22:19:20,941 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:20,942 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-31 22:19:20,942 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:20,942 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-31 22:19:20,942 INFO    SenderThread:1754882 [file_stream.py:finish():601] file stream finish called
+2024-12-31 22:19:21,224 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:19:21,286 INFO    SenderThread:1754882 [file_stream.py:finish():605] file stream finish is done
+2024-12-31 22:19:21,286 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 13
+2024-12-31 22:19:21,286 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:19:21,286 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:21,286 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-31 22:19:21,287 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:21,287 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-31 22:19:21,287 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 14
+2024-12-31 22:19:21,287 DEBUG   SenderThread:1754882 [sender.py:send():379] send: final
+2024-12-31 22:19:21,287 DEBUG   SenderThread:1754882 [sender.py:send():379] send: footer
+2024-12-31 22:19:21,287 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:21,287 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-31 22:19:21,287 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:21,287 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-31 22:19:21,288 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:19:21,288 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: server_info
+2024-12-31 22:19:21,288 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:19:21,289 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-31 22:19:21,289 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:19:21,289 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: server_info
+2024-12-31 22:19:21,291 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-31 22:19:21,292 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:21,337 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:19:21,338 INFO    MainThread:1754882 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-31 22:19:21,338 INFO    MainThread:1754882 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-31 22:19:21,338 INFO    MainThread:1754882 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-31 22:19:21,339 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-31 22:19:21,339 INFO    HandlerThread:1754882 [handler.py:finish():882] shutting down handler
+2024-12-31 22:19:22,289 INFO    WriterThread:1754882 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/run-gwnrzhmx.wandb
+2024-12-31 22:19:22,338 INFO    SenderThread:1754882 [sender.py:finish():1615] shutting down sender
+2024-12-31 22:19:22,338 INFO    SenderThread:1754882 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:19:22,338 INFO    SenderThread:1754882 [file_pusher.py:join():175] waiting for file pusher
diff --git a/gwnrzhmx/wandb/debug.log b/gwnrzhmx/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..c56e86f10e936bceaf9f4d792f4cfe2e811d65da
--- /dev/null
+++ b/gwnrzhmx/wandb/debug.log
@@ -0,0 +1,31 @@
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Configure stats pid to 1754682
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/settings
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/logs/debug.log
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/logs/debug-internal.log
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:init():569] calling init triggers
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx'}
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:init():619] starting backend
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:init():623] setting up manager
+2024-12-31 22:15:02,171 INFO    MainThread:1754682 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-31 22:15:02,172 INFO    MainThread:1754682 [wandb_init.py:init():631] backend started and connected
+2024-12-31 22:15:02,184 INFO    MainThread:1754682 [wandb_init.py:init():720] updated telemetry
+2024-12-31 22:15:02,192 INFO    MainThread:1754682 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-31 22:15:02,409 INFO    MainThread:1754682 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-31 22:15:02,476 INFO    MainThread:1754682 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-31 22:15:02,476 INFO    MainThread:1754682 [wandb_init.py:init():804] starting run threads in backend
+2024-12-31 22:15:02,835 INFO    MainThread:1754682 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-31 22:15:02,835 INFO    MainThread:1754682 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-31 22:15:02,835 INFO    MainThread:1754682 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-31 22:15:02,836 INFO    MainThread:1754682 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-31 22:15:02,837 INFO    MainThread:1754682 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-31 22:15:02,837 INFO    MainThread:1754682 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/tb, True
+2024-12-31 22:15:13,400 INFO    MainThread:1754682 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-31 22:19:22,339 WARNING MsgRouterThr:1754682 [router.py:message_loop():77] message_loop has been closed
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code/FISH/eval_robot.py b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..41791869dc3657b820d72a52a39ee14580eb36bd
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				"config.grasped_obj_name": grasped_obj_name,
+				"config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					total_num_episodes += filtered_run.summary_metrics['episode']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..aa0b4de8645081b21cdaf01f27a2a965637fa267
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1007_0'
+wandb_notes:
+  desc: null
+  value: 1007_0_restarted_2
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1735701302
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: hbm
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/diff.patch b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..109c7897cd7a3ddec63cb5f7b0a087c4f2410ed0
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/diff.patch
@@ -0,0 +1,53 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 0f6a9ec..b9e3dfe 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
++# wandb_run_id: '1002_0' 
++
++# all books, 6/20 demos per book
++# RGBD+mask+act history
+ wandb_run_id: '1007_0' 
+ 
++# RGBD+mask+contact(all ftrs)+act history
++# wandb_run_id: '1006_0'
++
++# RGBD+mask+contact(contact+DTC)+act history
++# wandb_run_id: '1745_0'
+ 
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 21e335c..71a5d1c 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -155,7 +155,12 @@ run_id_list = [
+     # '1022_0',
+     # '1023_0',
+     # '1007_0',
+-    '1007_0'
++    # '1007_0'
++]
++
++run_id_list = [
++    # '1006_0',
++    '1745_0'
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py
+index 3302c31..1946eca 100644
+--- a/gym-envs/gym_envs/envs/franka_env.py
++++ b/gym-envs/gym_envs/envs/franka_env.py
+@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env):
+ 
+ 	def mask_obs_callback(self, msg):
+ 		msg = self.possibly_resize_mask_obs(msg)
+-		rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}")
++		# rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}")
+ 		self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg)  
+ 
+ 	def contact_obs_callback(self, msg):
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..cc7e5e91447669fe989ab08b6e5b18934b0ea7ca
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.4, 0.04400911936343024, 0.8621948956718848, 0.05324718365067565, 0.803534984527916, 0.11712844677476308, 0.7487166672447589, 0.23282015328982517, 0.6637700199753331]]}
\ No newline at end of file
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..1d95a7bddc9f4697c0a8c2264aab5142c2121875
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[2, 5, 0.4, 0.04070010790474144, 0.8728086947649158, 0.09036684044447546, 0.8503108768954606, 0.12382420905007341, 0.7396230101106143, 0.1259237081560607, 0.7419417798035252]]}
\ No newline at end of file
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4 b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..2b316fcffdf854924907f3401a9fdf64996c3095
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c17a3e9a7bb8fc87a935a218fd8132e73eba1da41b97e1cd42c23ef5c1e26cf9
+size 1139182
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/1_eval_1_db2109c40f910f358754.mp4 b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/1_eval_1_db2109c40f910f358754.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..88443191ad26560f5dcced4d4c17533f91d94bae
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/1_eval_1_db2109c40f910f358754.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db2109c40f910f358754ba1da0965361ab3b4a6adb800a0207d1a7d0bdfd0d2b
+size 1217463
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4 b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..311c870bd14268ef235c44851561c05fd41e4754
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29c79964510fc68204d881dae01ac65d1e42ad65d896e0c56cd6bc77f781a4c0
+size 1214674
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4 b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..6b867ad7e408f8154de6b0937a0aa9e3b370d1ab
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d71847ccf69c2b06c3e663657321a935a91869ddcf855e7e53d2eea24d3f63f
+size 1176392
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4 b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..36927230fc92514f533a982f8443eb49c3957b70
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a9cd71b9b9e63fcffc667c82fce584257e4da0cf58a858b45ec50af0c8d6fc4
+size 1223048
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..3b0563cb5e51d301c90a96c17e11100a01e3783a
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
@@ -0,0 +1,270 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1735701313.401082]: resetting environment
+[INFO] [1735701313.403883]: cleared current plan
+[INFO] [1735701313.404884]: moving to home
+[INFO] [1735701314.406905]: reached home
+[INFO] [1735701314.408356]: reset action history
+[INFO] [1735701317.117715]: environment reset
+Starting episode 0
+[INFO] [1735701317.120153]: resetting environment
+[INFO] [1735701317.123147]: cleared current plan
+[INFO] [1735701317.124639]: moving to home
+[INFO] [1735701318.127417]: reached home
+[INFO] [1735701318.129242]: reset action history
+[INFO] [1735701320.839267]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1735701327.048404]: resetting environment
+[INFO] [1735701327.051494]: cleared current plan
+[INFO] [1735701327.052844]: moving to home
+[INFO] [1735701328.055471]: reached home
+[INFO] [1735701328.056871]: reset action history
+[INFO] [1735701330.764729]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/episode_rosbags/episode_0_2024-12-31-22-15-30.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1754682.log
+started roslaunch server http://158.130.50.37:35769/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [1754982]
+process[print_text-2]: started with pid [1754983]
+process[rosbag_record-3]: started with pid [1755007]
+started bagging!
+For topic gripper_width: timestamp difference is -227163204 for nearest: 1735701333353334061 - target: 1735701333580497265 at idx 40
+For topic gripper_width: timestamp difference is -254785834 for nearest: 1735701340453334035 - target: 1735701340708119869 at idx 124
+[INFO] [1735701357.774065]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1735701358.524072]: Stored episode 1.
+[INFO] [1735701358.524383]: Saving video...
+[INFO] [1735701358.711926]: Video saved!
+Starting episode 1
+[INFO] [1735701358.731515]: resetting environment
+[INFO] [1735701358.736107]: cleared current plan
+[INFO] [1735701358.736325]: moving to home
+[INFO] [1735701362.139145]: reached home
+[INFO] [1735701362.139950]: reset action history
+[INFO] [1735701364.851976]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735701366.822771]: resetting environment
+[INFO] [1735701366.828488]: cleared current plan
+[INFO] [1735701366.828655]: moving to home
+[INFO] [1735701367.829211]: reached home
+[INFO] [1735701367.829637]: reset action history
+[INFO] [1735701370.536996]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/episode_rosbags/episode_1_2024-12-31-22-16-10.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1754682.log
+started roslaunch server http://158.130.50.37:43853/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [1755141]
+process[print_text-5]: started with pid [1755142]
+process[rosbag_record-6]: started with pid [1755143]
+started bagging!
+For topic gripper_width: timestamp difference is 105041066 for nearest: 1735701373520002404 - target: 1735701373414961338 at idx 39
+For topic gripper_width: timestamp difference is 106445118 for nearest: 1735701375320001408 - target: 1735701375213556290 at idx 62
+For topic gripper_width: timestamp difference is -179696639 for nearest: 1735701392253334443 - target: 1735701392433031082 at idx 249
+[rosbag_record-6] killing on exitng episode...is -179696639 for nearest: 1735701392253334443 - target: 1735701392433031082 at idx 249
+[rosbag_record-6] killing on exitng episode...is -179696639 for nearest: 1735701392253334443 - target: 1735701392433031082 at idx 249
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1735701399.488221]: Stored episode 2.
+[INFO] [1735701399.489337]: Saving video...
+[INFO] [1735701399.709658]: Video saved!
+Starting episode 2
+[INFO] [1735701399.741094]: resetting environment
+[INFO] [1735701399.756597]: cleared current plan
+[INFO] [1735701399.758631]: moving to home
+[INFO] [1735701402.961206]: reached home
+[INFO] [1735701402.962026]: reset action history
+[INFO] [1735701405.685345]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735701406.972410]: resetting environment
+[INFO] [1735701406.986502]: cleared current plan
+[INFO] [1735701406.987519]: moving to home
+[INFO] [1735701407.990720]: reached home
+[INFO] [1735701407.999761]: reset action history
+[INFO] [1735701410.770917]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/episode_rosbags/episode_2_2024-12-31-22-16-50.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1754682.log
+started roslaunch server http://158.130.50.37:32881/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [1755287]
+process[print_text-8]: started with pid [1755288]
+process[rosbag_record-9]: started with pid [1755289]
+started bagging!
+[INFO] [1735701439.107686]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1735701439.846486]: Stored episode 3.
+[INFO] [1735701439.849311]: Saving video...
+[INFO] [1735701440.080177]: Video saved!
+Starting episode 3
+[INFO] [1735701440.139168]: resetting environment
+[INFO] [1735701440.155370]: cleared current plan
+[INFO] [1735701440.157501]: moving to home
+[INFO] [1735701443.581214]: reached home
+[INFO] [1735701443.587829]: reset action history
+[INFO] [1735701446.368345]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735701454.740094]: resetting environment
+[INFO] [1735701454.757198]: cleared current plan
+[INFO] [1735701454.765368]: moving to home
+[INFO] [1735701455.769991]: reached home
+[INFO] [1735701455.770585]: reset action history
+[INFO] [1735701458.548664]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/episode_rosbags/episode_3_2024-12-31-22-17-38.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1754682.log
+started roslaunch server http://158.130.50.37:46043/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [1755437]
+process[print_text-11]: started with pid [1755438]
+process[rosbag_record-12]: started with pid [1755439]
+started bagging!
+For topic gripper_width: timestamp difference is -224299821 for nearest: 1735701460753336516 - target: 1735701460977636337 at idx 29
+[INFO] [1735701486.647630]: Storing episode...
+[rosbag_record-12] killing on exit[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1735701487.440789]: Stored episode 4.
+[INFO] [1735701487.442186]: Saving video...
+[INFO] [1735701487.655361]: Video saved!
+Starting episode 4
+[INFO] [1735701487.709635]: resetting environment
+[INFO] [1735701487.728774]: cleared current plan
+[INFO] [1735701487.733654]: moving to home
+[INFO] [1735701491.246502]: reached home
+[INFO] [1735701491.258841]: reset action history
+[INFO] [1735701494.055734]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735701495.993594]: resetting environment
+[INFO] [1735701496.007232]: cleared current plan
+[INFO] [1735701496.009448]: moving to home
+[INFO] [1735701497.023480]: reached home
+[INFO] [1735701497.027369]: reset action history
+[INFO] [1735701499.806353]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/episode_rosbags/episode_4_2024-12-31-22-18-19.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1754682.log
+started roslaunch server http://158.130.50.37:39407/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [1755571]
+process[print_text-14]: started with pid [1755572]
+process[rosbag_record-15]: started with pid [1755573]
+started bagging!
+For topic gripper_width: timestamp difference is -106473054 for nearest: 1735701524353363191 - target: 1735701524459836245 at idx 249
+For topic gripper_width: timestamp difference is -104893229 for nearest: 1735701526353286960 - target: 1735701526458180189 at idx 249
+[INFO] [1735701529.130116]: Storing episode...
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit
+[print_text-14] killing on exit
+[INFO] [1735701529.919932]: Stored episode 5.
+[INFO] [1735701529.938460]: Saving video...
+[INFO] [1735701530.163805]: Video saved!
+[INFO] [1735701530.208915]: resetting environment
+[INFO] [1735701530.231459]: cleared current plan
+[INFO] [1735701530.235298]: moving to home
+[INFO] [1735701533.745373]: reached home
+[INFO] [1735701533.749176]: reset action history
+[INFO] [1735701536.612439]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/requirements.txt b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-metadata.json b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..6873e1b721a0871260da784c52d088c12c2c2ac3
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-01T03:15:02.500360",
+    "startedAt": "2025-01-01T03:15:02.164240",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3748.02675,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.012,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4497.046,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4495.794,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 535.0275993347168
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..5edd76027578bbd3268919391d32efbf734117da
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "c17a3e9a7bb8fc87a935a218fd8132e73eba1da41b97e1cd42c23ef5c1e26cf9", "size": 1139182, "path": "media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4"}, "global_step": 664, "_timestamp": 1735701540.332067, "_runtime": 238.1593780517578, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "db2109c40f910f358754ba1da0965361ab3b4a6adb800a0207d1a7d0bdfd0d2b", "size": 1217463, "path": "media/videos/eval/1_eval_1_db2109c40f910f358754.mp4"}, "eval/num_success": 2.0, "episode": 5.0, "eval/success_rate": 0.4000000059604645, "eval/success_rate_lb_0.01": 0.044009119272232056, "eval/success_rate_ub_0.01": 0.8621948957443237, "eval/success_rate_lb_0.025": 0.05324718356132507, "eval/success_rate_ub_0.025": 0.803534984588623, "eval/success_rate_lb_0.05": 0.11712844669818878, "eval/success_rate_ub_0.05": 0.7487166523933411, "eval/success_rate_lb_0.1": 0.23282015323638916, "eval/success_rate_ub_0.1": 0.6637700200080872, "eval/2_eval": {"_type": "video-file", "sha256": "29c79964510fc68204d881dae01ac65d1e42ad65d896e0c56cd6bc77f781a4c0", "size": 1214674, "path": "media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "4d71847ccf69c2b06c3e663657321a935a91869ddcf855e7e53d2eea24d3f63f", "size": 1176392, "path": "media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "7a9cd71b9b9e63fcffc667c82fce584257e4da0cf58a858b45ec50af0c8d6fc4", "size": 1223048, "path": "media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "85c54431b4eb7c8145a72a6437cbffccb84a9cf38051f18b07dd0a638d459f25", "size": 401, "artifact_path": "wandb-client-artifact://5hksy63xmymf2fiev46b5rkzgivke8blt5jhxwm73buqzhanmc78ths5u8qcqir9gxmynb3l7659ri2zwqtelj6bjn1jk3mqt8tmhvns5v3a4f46zkeao2qwjfiwzcij/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zkiyvtm41x2pe388ubmdo5jis2e57pdjt9vbsjssm7lc7uvzkgfm4nccr30vriw7dgq2la8a7h8gq5zoj8u0ihrxgc1xz4dyzsmd5oo1mddby2nbo73ktgchlb72fd0o:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 2, "total_num_episodes": 5, "total_success_rate": 0.4, "total_success_rate_lb_0.01": 0.04070010790474144, "total_success_rate_ub_0.01": 0.8728086947649158, "total_success_rate_lb_0.025": 0.09036684044447546, "total_success_rate_ub_0.025": 0.8503108768954606, "total_success_rate_lb_0.05": 0.12382420905007341, "total_success_rate_ub_0.05": 0.7396230101106143, "total_success_rate_lb_0.1": 0.1259237081560607, "total_success_rate_ub_0.1": 0.7419417798035252, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "d5fa38cba1266bd8cfebdb65df17fa0d0fd3a3a64e30f6512f69c2b67c0ef0fc", "size": 506, "artifact_path": "wandb-client-artifact://wx51yvkwvd6uoi7gekp3qevfauqi1nhzquqdpoikupjhhhcvhanoqwejh63a2siqfokpes8tnyf4r91op5xlsmp39bb4zf6vmqrgu1pcxrmegg9vp6u0kpg73r6yhbna/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://r2hzrbyyt3yob28t7zymmp923wux5kqqguq7ujsinekubktis7cpw5q77v36bcqbcu7yuvuwo9oe7wgoydvtl6a65kxhep2dvbqgo4b97z6tpm8fiwau9tl34jwi5n65:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 248}, "eval/total_success_rate": 0.4000000059604645, "eval/total_success_rate_lb_0.01": 0.04070010781288147, "eval/total_success_rate_ub_0.01": 0.8728086948394775, "eval/total_success_rate_lb_0.025": 0.09036684036254883, "eval/total_success_rate_ub_0.025": 0.8503108620643616, "eval/total_success_rate_lb_0.05": 0.12382420897483826, "eval/total_success_rate_ub_0.05": 0.7396230101585388, "eval/total_success_rate_lb_0.1": 0.12592370808124542, "eval/total_success_rate_ub_0.1": 0.7419417500495911}
\ No newline at end of file
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/logs/debug-internal.log b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..12bd616b6c898ca0ed2598a9c890991267a2e2cd
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/logs/debug-internal.log
@@ -0,0 +1,748 @@
+2024-12-31 22:15:02,172 INFO    StreamThr :1754882 [internal.py:wandb_internal():85] W&B internal server running at pid: 1754882, started at: 2024-12-31 22:15:02.171894
+2024-12-31 22:15:02,173 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status
+2024-12-31 22:15:02,185 INFO    WriterThread:1754882 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/run-gwnrzhmx.wandb
+2024-12-31 22:15:02,185 DEBUG   SenderThread:1754882 [sender.py:send():379] send: header
+2024-12-31 22:15:02,195 DEBUG   SenderThread:1754882 [sender.py:send():379] send: run
+2024-12-31 22:15:02,394 INFO    SenderThread:1754882 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files
+2024-12-31 22:15:02,394 INFO    SenderThread:1754882 [sender.py:_start_run_threads():1188] run started: gwnrzhmx with start time 1735701302.172689
+2024-12-31 22:15:02,410 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: check_version
+2024-12-31 22:15:02,410 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: check_version
+2024-12-31 22:15:02,481 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: run_start
+2024-12-31 22:15:02,486 DEBUG   HandlerThread:1754882 [system_info.py:__init__():26] System info init
+2024-12-31 22:15:02,486 DEBUG   HandlerThread:1754882 [system_info.py:__init__():41] System info init done
+2024-12-31 22:15:02,486 INFO    HandlerThread:1754882 [system_monitor.py:start():194] Starting system monitor
+2024-12-31 22:15:02,486 INFO    SystemMonitor:1754882 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-31 22:15:02,486 INFO    HandlerThread:1754882 [system_monitor.py:probe():214] Collecting system info
+2024-12-31 22:15:02,486 INFO    SystemMonitor:1754882 [interfaces.py:start():188] Started cpu monitoring
+2024-12-31 22:15:02,487 INFO    SystemMonitor:1754882 [interfaces.py:start():188] Started disk monitoring
+2024-12-31 22:15:02,488 INFO    SystemMonitor:1754882 [interfaces.py:start():188] Started gpu monitoring
+2024-12-31 22:15:02,488 INFO    SystemMonitor:1754882 [interfaces.py:start():188] Started memory monitoring
+2024-12-31 22:15:02,488 INFO    SystemMonitor:1754882 [interfaces.py:start():188] Started network monitoring
+2024-12-31 22:15:02,500 DEBUG   HandlerThread:1754882 [system_info.py:probe():152] Probing system
+2024-12-31 22:15:02,501 DEBUG   HandlerThread:1754882 [system_info.py:_probe_git():137] Probing git
+2024-12-31 22:15:02,506 DEBUG   HandlerThread:1754882 [system_info.py:_probe_git():145] Probing git done
+2024-12-31 22:15:02,506 DEBUG   HandlerThread:1754882 [system_info.py:probe():200] Probing system done
+2024-12-31 22:15:02,506 DEBUG   HandlerThread:1754882 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T03:15:02.500360', 'startedAt': '2025-01-01T03:15:02.164240', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.02675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.012, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.046, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4495.794, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 535.0275993347168}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-31 22:15:02,506 INFO    HandlerThread:1754882 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-31 22:15:02,506 INFO    HandlerThread:1754882 [system_monitor.py:probe():227] Publishing system info
+2024-12-31 22:15:02,506 DEBUG   HandlerThread:1754882 [system_info.py:_save_code():44] Saving code
+2024-12-31 22:15:02,508 DEBUG   HandlerThread:1754882 [system_info.py:_save_code():67] Saving code done
+2024-12-31 22:15:02,508 DEBUG   HandlerThread:1754882 [system_info.py:_save_patches():84] Saving git patches
+2024-12-31 22:15:02,570 DEBUG   HandlerThread:1754882 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-31 22:15:02,570 INFO    HandlerThread:1754882 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-31 22:15:02,573 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:15:02,573 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-31 22:15:02,574 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-31 22:15:02,574 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-31 22:15:02,834 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-31 22:15:02,834 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: python_packages
+2024-12-31 22:15:02,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:02,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:15:02,836 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:15:02,842 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/6oatoedz-wandb-metadata.json
+2024-12-31 22:15:02,854 INFO    wandb-upload_2:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/wizkghbd-diff.patch
+2024-12-31 22:15:02,877 INFO    HandlerThread:1754882 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/tb"
+  save: true
+}
+control {
+  relay_id: "25a480f1578d4ed69d39e4d6a73bba29"
+}
+_info {
+  stream_id: "gwnrzhmx"
+}
+
+2024-12-31 22:15:02,877 DEBUG   HandlerThread:1754882 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-31 22:15:02,902 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/adxeg02d-code/FISH/eval_robot.py
+2024-12-31 22:15:02,917 DEBUG   SenderThread:1754882 [sender.py:send():379] send: telemetry
+2024-12-31 22:15:02,929 DEBUG   SenderThread:1754882 [sender.py:send():379] send: tbrecord
+2024-12-31 22:15:02,929 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:15:02,930 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0 with policy live
+2024-12-31 22:15:03,131 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/swkfhmz1-events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:15:03,395 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code/FISH/eval_robot.py
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-metadata.json
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/diff.patch
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/requirements.txt
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code
+2024-12-31 22:15:03,396 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code/FISH
+2024-12-31 22:15:03,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:04,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:05,395 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:05,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:06,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:07,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:07,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:08,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:09,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:10,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:11,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:12,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:12,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:13,401 DEBUG   SenderThread:1754882 [sender.py:send():379] send: config
+2024-12-31 22:15:13,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:14,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:15,398 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:15,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:16,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:17,398 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:17,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:15:17,835 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:15:17,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:17,891 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:18,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:19,399 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:19,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:20,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:21,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:22,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:23,400 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:23,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:23,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:24,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:25,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:26,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:27,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:28,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:29,097 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:29,402 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:29,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:30,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:31,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:32,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:15:32,835 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:15:32,877 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:33,403 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:33,839 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:34,733 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:34,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:35,403 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:35,403 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml
+2024-12-31 22:15:35,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:36,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:37,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:38,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:39,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:40,797 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:40,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:41,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:42,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:43,405 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:43,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:44,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:45,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:45,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:46,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:47,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:15:47,835 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:15:47,877 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:48,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:49,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:50,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:50,973 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:51,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:52,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:53,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:54,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:55,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:56,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:15:56,845 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:57,843 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:58,726 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:15:58,726 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4 with policy now
+2024-12-31 22:15:58,729 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:15:58,730 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:15:58,730 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:15:58,731 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:15:58,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:15:59,012 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/0uoo3djq-media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4
+2024-12-31 22:15:59,410 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:15:59,410 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4
+2024-12-31 22:15:59,411 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:15:59,411 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval
+2024-12-31 22:15:59,411 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos
+2024-12-31 22:15:59,411 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media
+2024-12-31 22:15:59,692 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/0wj9cy4r-media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4
+2024-12-31 22:15:59,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:00,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:01,411 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:01,845 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:02,139 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:02,489 DEBUG   SystemMonitor:1754882 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-31 22:16:02,489 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:16:02,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:16:02,836 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:16:02,845 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:03,412 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:03,845 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:04,849 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:05,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:06,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:07,413 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:07,413 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:16:07,665 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/1rsoj03n-events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:16:07,829 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:07,877 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:08,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:09,414 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:09,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:10,845 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:11,850 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:12,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:12,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:13,415 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:13,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:14,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:15,416 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:15,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:16,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:17,416 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:17,835 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:16:17,836 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:16:17,877 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:17,960 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:18,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:19,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:20,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:21,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:22,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:23,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:23,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:24,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:25,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:26,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:27,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:28,838 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:28,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:29,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:30,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:31,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:32,490 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:16:32,836 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:16:32,836 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:16:32,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:33,421 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:33,856 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:33,987 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:34,856 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:35,422 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:35,856 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:36,856 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:37,856 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:38,857 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:39,489 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:39,737 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:16:39,737 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_db2109c40f910f358754.mp4 with policy now
+2024-12-31 22:16:39,738 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:16:39,738 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:16:39,738 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:16:39,739 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:16:39,857 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:40,009 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/urlv7djh-media/videos/eval/1_eval_1_db2109c40f910f358754.mp4
+2024-12-31 22:16:40,424 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:16:40,424 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/1_eval_1_db2109c40f910f358754.mp4
+2024-12-31 22:16:40,424 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval
+2024-12-31 22:16:40,733 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/52bxirtc-media/videos/eval/1_eval_1_db2109c40f910f358754.mp4
+2024-12-31 22:16:40,861 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:41,424 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:41,858 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:42,860 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:43,859 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:44,859 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:45,005 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:45,425 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:45,859 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:46,861 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:47,274 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:16:47,274 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:16:47,274 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:16:47,426 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:47,426 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:16:47,426 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:16:47,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:16:47,840 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:16:47,863 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:48,865 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:49,427 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:49,864 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:50,033 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:50,864 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:51,861 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:52,862 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:53,428 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:16:53,862 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:54,868 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:55,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:16:55,867 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:56,867 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:57,867 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:58,867 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:16:59,868 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:00,868 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:01,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:01,870 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:02,490 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:17:02,837 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:17:02,838 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:17:02,881 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:03,873 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:04,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:05,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:06,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:06,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:07,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:08,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:09,871 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:10,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:11,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:12,840 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:12,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:13,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:14,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:15,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:16,872 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:17,841 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:17,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:17:17,845 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:17:17,885 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:18,879 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:19,882 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:20,128 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:17:20,128 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4 with policy now
+2024-12-31 22:17:20,138 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:17:20,139 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:17:20,139 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:17:20,139 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:17:20,391 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/kzf77wj5-media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4
+2024-12-31 22:17:20,437 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:17:20,437 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4
+2024-12-31 22:17:20,437 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval
+2024-12-31 22:17:20,705 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/mqgrvrh4-media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4
+2024-12-31 22:17:20,884 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:21,437 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:21,887 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:22,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:22,892 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:23,888 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:24,890 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:25,439 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:25,887 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:26,892 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:27,440 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:27,850 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:27,889 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:28,890 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:29,896 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:30,893 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:31,894 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:32,491 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:17:32,853 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:17:32,853 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:17:32,901 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:32,933 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:33,889 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:34,893 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:35,113 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:17:35,113 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:17:35,114 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:17:35,442 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:17:35,442 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:17:35,727 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/4uw8m5po-events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:17:35,894 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:36,901 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:37,443 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:37,901 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:38,549 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:38,898 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:39,443 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:39,906 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:40,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:41,444 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:41,901 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:42,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:43,445 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:17:43,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:43,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:44,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:45,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:46,902 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:47,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:17:47,842 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:17:47,903 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:48,903 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:48,981 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:49,903 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:50,903 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:51,903 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:52,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:53,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:54,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:54,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:55,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:56,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:57,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:58,904 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:17:59,846 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:17:59,905 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:00,905 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:01,905 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:02,491 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:18:02,842 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:18:02,842 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:18:02,905 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:03,905 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:04,908 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:05,002 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:05,908 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:06,911 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:07,698 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:18:07,698 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4 with policy now
+2024-12-31 22:18:07,704 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:18:07,704 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:18:07,705 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:18:07,707 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:18:07,916 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:07,982 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/1rq1l2ra-media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4
+2024-12-31 22:18:08,453 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:18:08,453 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4
+2024-12-31 22:18:08,453 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval
+2024-12-31 22:18:08,750 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/4sqoioh6-media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4
+2024-12-31 22:18:08,916 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:09,453 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:09,916 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:10,854 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:10,915 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:11,929 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:12,927 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:13,455 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:13,932 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:14,918 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:15,455 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:15,930 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:15,960 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:16,165 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:18:16,165 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:18:16,165 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:18:16,456 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:18:16,456 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:18:16,918 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:17,456 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:17,855 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:18:17,855 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:18:17,929 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:18,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:19,457 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:19,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:20,927 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:21,458 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:21,698 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:21,922 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:22,922 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:23,458 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:23,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:24,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:25,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:26,849 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:26,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:27,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:28,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:29,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:30,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:31,923 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:32,492 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:18:32,493 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:32,843 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:18:32,843 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:18:32,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:33,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:34,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:35,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:36,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:37,850 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:37,924 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:38,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:39,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:40,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:41,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:42,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:43,850 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:43,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:44,925 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:45,466 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:45,926 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:46,926 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:47,467 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:47,843 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:18:47,843 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:18:47,926 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:48,906 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:48,926 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:49,949 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:50,213 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:18:50,213 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:18:50,213 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4 with policy now
+2024-12-31 22:18:50,214 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:18:50,214 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:18:50,214 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:18:50,468 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:18:50,468 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4
+2024-12-31 22:18:50,468 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval
+2024-12-31 22:18:50,472 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/8k6l9wqp-media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4
+2024-12-31 22:18:50,732 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/zy1m3qkj-media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4
+2024-12-31 22:18:50,942 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:51,468 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:51,934 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:52,942 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:53,942 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:54,789 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:18:54,935 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:55,469 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:55,940 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:56,944 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:57,470 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:18:57,959 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:59,009 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:18:59,216 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:18:59,216 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:18:59,217 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:18:59,471 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:18:59,471 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:18:59,714 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/xu2xzo59-events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:18:59,892 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:18:59,892 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:18:59,949 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:00,073 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:19:00,347 INFO    wandb-upload_0:1754882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp79jlbk65
+2024-12-31 22:19:00,469 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:19:00,471 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:19:00,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json
+2024-12-31 22:19:00,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media
+2024-12-31 22:19:00,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table
+2024-12-31 22:19:00,473 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval
+2024-12-31 22:19:00,477 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:19:00,740 INFO    SenderThread:1754882 [sender.py:send_request_log_artifact():1525] logged artifact run-gwnrzhmx-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE1NjEwNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MjI1Ng==', 'latestArtifact': None}}
+2024-12-31 22:19:00,740 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:19:00,740 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now
+2024-12-31 22:19:00,742 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:19:00,742 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,742 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:19:00,743 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,743 DEBUG   SenderThread:1754882 [sender.py:send():379] send: telemetry
+2024-12-31 22:19:00,743 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,743 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,743 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,744 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,744 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,744 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,744 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,745 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,745 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,745 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,745 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,746 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,746 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,746 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,746 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,746 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,747 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,747 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,747 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,747 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,748 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:00,748 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:00,748 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:19:00,946 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:00,949 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/2nfsdqaf-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json
+2024-12-31 22:19:01,100 INFO    wandb-upload_0:1754882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3mt8271m
+2024-12-31 22:19:01,197 INFO    wandb-upload_2:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/9olnewwt-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json
+2024-12-31 22:19:01,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:19:01,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:19:01,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json
+2024-12-31 22:19:01,473 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval
+2024-12-31 22:19:01,529 INFO    SenderThread:1754882 [sender.py:send_request_log_artifact():1525] logged artifact run-gwnrzhmx-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE1NjExMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MjI1Nw==', 'latestArtifact': None}}
+2024-12-31 22:19:01,529 DEBUG   SenderThread:1754882 [sender.py:send():379] send: files
+2024-12-31 22:19:01,529 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json with policy now
+2024-12-31 22:19:01,531 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:19:01,531 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:01,532 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:01,752 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/zc6s33tb-media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json
+2024-12-31 22:19:01,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:02,000 INFO    wandb-upload_2:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/x16wgeuo-media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json
+2024-12-31 22:19:02,472 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:19:02,492 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:19:02,844 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:19:02,844 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:19:02,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:03,473 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:19:03,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:04,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:05,943 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:19:05,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:06,945 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:07,947 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:08,946 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:09,948 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:10,946 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:11,173 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:19:11,221 DEBUG   SenderThread:1754882 [sender.py:send():379] send: exit
+2024-12-31 22:19:11,221 INFO    SenderThread:1754882 [sender.py:send_exit():586] handling exit code: 0
+2024-12-31 22:19:11,221 INFO    SenderThread:1754882 [sender.py:send_exit():588] handling runtime: 248
+2024-12-31 22:19:11,223 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:11,223 INFO    SenderThread:1754882 [sender.py:send_exit():594] send defer
+2024-12-31 22:19:11,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:11,223 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-31 22:19:11,223 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:11,223 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-31 22:19:11,223 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 1
+2024-12-31 22:19:11,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:11,223 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-31 22:19:11,223 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:11,223 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-31 22:19:11,224 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 2
+2024-12-31 22:19:11,224 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:11,224 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-31 22:19:11,224 INFO    HandlerThread:1754882 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-31 22:19:11,224 DEBUG   SystemMonitor:1754882 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-31 22:19:11,224 DEBUG   SystemMonitor:1754882 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-31 22:19:11,225 INFO    HandlerThread:1754882 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-31 22:19:11,225 INFO    HandlerThread:1754882 [interfaces.py:finish():200] Joined disk monitor
+2024-12-31 22:19:11,276 INFO    HandlerThread:1754882 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-31 22:19:11,276 INFO    HandlerThread:1754882 [interfaces.py:finish():200] Joined memory monitor
+2024-12-31 22:19:11,276 INFO    HandlerThread:1754882 [interfaces.py:finish():200] Joined network monitor
+2024-12-31 22:19:11,277 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:11,277 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-31 22:19:11,277 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 3
+2024-12-31 22:19:11,277 DEBUG   SenderThread:1754882 [sender.py:send():379] send: stats
+2024-12-31 22:19:11,277 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:11,277 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-31 22:19:11,277 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:11,277 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-31 22:19:11,277 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 4
+2024-12-31 22:19:11,278 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:11,278 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-31 22:19:11,476 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:19:13,477 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:19:18,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:19:18,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:19:18,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-31 22:19:18,224 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:18,224 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-31 22:19:18,224 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 5
+2024-12-31 22:19:18,224 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:19:18,224 DEBUG   SenderThread:1754882 [sender.py:send():379] send: history
+2024-12-31 22:19:18,224 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:19:18,225 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:18,225 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-31 22:19:18,225 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:18,226 DEBUG   SenderThread:1754882 [sender.py:send():379] send: summary
+2024-12-31 22:19:18,229 INFO    SenderThread:1754882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:19:18,229 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:18,230 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-31 22:19:18,230 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 6
+2024-12-31 22:19:18,230 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:18,230 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-31 22:19:18,230 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:18,230 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-31 22:19:18,233 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:19:18,352 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 7
+2024-12-31 22:19:18,352 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:18,352 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-31 22:19:18,352 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:18,352 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-31 22:19:18,479 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml
+2024-12-31 22:19:18,479 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:19:19,193 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 8
+2024-12-31 22:19:19,193 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:19,193 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-31 22:19:19,193 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:19,194 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-31 22:19:19,194 INFO    SenderThread:1754882 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-31 22:19:19,196 INFO    SenderThread:1754882 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-31 22:19:19,198 INFO    SenderThread:1754882 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-31 22:19:19,208 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 9
+2024-12-31 22:19:19,208 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:19,208 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-31 22:19:19,209 DEBUG   SenderThread:1754882 [sender.py:send():379] send: artifact
+2024-12-31 22:19:19,223 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:19:19,479 INFO    Thread-12 :1754882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:19:19,540 INFO    wandb-upload_0:1754882 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpio3ffp0z
+2024-12-31 22:19:19,540 INFO    wandb-upload_2:1754882 [upload_job.py:push():85] Skipped uploading /tmp/tmpcq73m_vk/wandb-job.json
+2024-12-31 22:19:19,648 INFO    wandb-upload_1:1754882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphqylxoc7
+2024-12-31 22:19:20,228 INFO    SenderThread:1754882 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE1NjI2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE0MjIyOA==', 'versionIndex': 394}}}
+2024-12-31 22:19:20,228 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:20,228 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-31 22:19:20,228 INFO    SenderThread:1754882 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-31 22:19:20,480 INFO    SenderThread:1754882 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files
+2024-12-31 22:19:20,480 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml config.yaml
+2024-12-31 22:19:20,480 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json wandb-summary.json
+2024-12-31 22:19:20,480 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0 events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:19:20,481 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/requirements.txt requirements.txt
+2024-12-31 22:19:20,485 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-metadata.json wandb-metadata.json
+2024-12-31 22:19:20,485 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log output.log
+2024-12-31 22:19:20,486 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/diff.patch diff.patch
+2024-12-31 22:19:20,486 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-31 22:19:20,486 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/1_eval_1_db2109c40f910f358754.mp4 media/videos/eval/1_eval_1_db2109c40f910f358754.mp4
+2024-12-31 22:19:20,490 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4 media/videos/eval/4_eval_4_7a9cd71b9b9e63fcffc6.mp4
+2024-12-31 22:19:20,491 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4 media/videos/eval/2_eval_2_29c79964510fc68204d8.mp4
+2024-12-31 22:19:20,491 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4 media/videos/eval/0_eval_0_c17a3e9a7bb8fc87a935.mp4
+2024-12-31 22:19:20,491 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4 media/videos/eval/3_eval_3_4d71847ccf69c2b06c3e.mp4
+2024-12-31 22:19:20,493 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json
+2024-12-31 22:19:20,493 INFO    SenderThread:1754882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json media/table/eval/total_success_rate_ci_6_d5fa38cba1266bd8cfeb.table.json
+2024-12-31 22:19:20,493 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 10
+2024-12-31 22:19:20,493 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:19:20,493 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:20,493 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-31 22:19:20,494 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:20,494 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-31 22:19:20,494 INFO    SenderThread:1754882 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:19:20,698 INFO    wandb-upload_0:1754882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/config.yaml
+2024-12-31 22:19:20,721 INFO    wandb-upload_3:1754882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/requirements.txt
+2024-12-31 22:19:20,725 INFO    wandb-upload_4:1754882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/output.log
+2024-12-31 22:19:20,735 INFO    wandb-upload_1:1754882 [upload_job.py:push():130] Uploaded file /tmp/tmpp7donv14wandb/vavfk9xn-events.out.tfevents.1735701302.leonmkim-ROG-Strix-G15CS-G15CS.1754682.0
+2024-12-31 22:19:20,739 INFO    wandb-upload_2:1754882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/files/wandb-summary.json
+2024-12-31 22:19:20,939 INFO    Thread-11 (_thread_body):1754882 [sender.py:transition_state():614] send defer: 11
+2024-12-31 22:19:20,940 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:20,940 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-31 22:19:20,940 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:20,940 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-31 22:19:20,940 INFO    SenderThread:1754882 [file_pusher.py:join():175] waiting for file pusher
+2024-12-31 22:19:20,941 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 12
+2024-12-31 22:19:20,941 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:20,942 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-31 22:19:20,942 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:20,942 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-31 22:19:20,942 INFO    SenderThread:1754882 [file_stream.py:finish():601] file stream finish called
+2024-12-31 22:19:21,224 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:19:21,286 INFO    SenderThread:1754882 [file_stream.py:finish():605] file stream finish is done
+2024-12-31 22:19:21,286 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 13
+2024-12-31 22:19:21,286 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:19:21,286 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:21,286 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-31 22:19:21,287 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:21,287 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-31 22:19:21,287 INFO    SenderThread:1754882 [sender.py:transition_state():614] send defer: 14
+2024-12-31 22:19:21,287 DEBUG   SenderThread:1754882 [sender.py:send():379] send: final
+2024-12-31 22:19:21,287 DEBUG   SenderThread:1754882 [sender.py:send():379] send: footer
+2024-12-31 22:19:21,287 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:19:21,287 INFO    HandlerThread:1754882 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-31 22:19:21,287 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:19:21,287 INFO    SenderThread:1754882 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-31 22:19:21,288 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:19:21,288 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: server_info
+2024-12-31 22:19:21,288 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:19:21,289 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-31 22:19:21,289 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:19:21,289 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: server_info
+2024-12-31 22:19:21,291 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-31 22:19:21,292 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:19:21,337 DEBUG   SenderThread:1754882 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:19:21,338 INFO    MainThread:1754882 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-31 22:19:21,338 INFO    MainThread:1754882 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-31 22:19:21,338 INFO    MainThread:1754882 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-31 22:19:21,339 DEBUG   HandlerThread:1754882 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-31 22:19:21,339 INFO    HandlerThread:1754882 [handler.py:finish():882] shutting down handler
+2024-12-31 22:19:22,289 INFO    WriterThread:1754882 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/run-gwnrzhmx.wandb
+2024-12-31 22:19:22,338 INFO    SenderThread:1754882 [sender.py:finish():1615] shutting down sender
+2024-12-31 22:19:22,338 INFO    SenderThread:1754882 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:19:22,338 INFO    SenderThread:1754882 [file_pusher.py:join():175] waiting for file pusher
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/logs/debug.log b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..c56e86f10e936bceaf9f4d792f4cfe2e811d65da
--- /dev/null
+++ b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/logs/debug.log
@@ -0,0 +1,31 @@
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Configure stats pid to 1754682
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/settings
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/logs/debug.log
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/logs/debug-internal.log
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:init():569] calling init triggers
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1007_0', 'wandb_notes': '1007_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx'}
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:init():619] starting backend
+2024-12-31 22:15:02,166 INFO    MainThread:1754682 [wandb_init.py:init():623] setting up manager
+2024-12-31 22:15:02,171 INFO    MainThread:1754682 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-31 22:15:02,172 INFO    MainThread:1754682 [wandb_init.py:init():631] backend started and connected
+2024-12-31 22:15:02,184 INFO    MainThread:1754682 [wandb_init.py:init():720] updated telemetry
+2024-12-31 22:15:02,192 INFO    MainThread:1754682 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-31 22:15:02,409 INFO    MainThread:1754682 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-31 22:15:02,476 INFO    MainThread:1754682 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-31 22:15:02,476 INFO    MainThread:1754682 [wandb_init.py:init():804] starting run threads in backend
+2024-12-31 22:15:02,835 INFO    MainThread:1754682 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-31 22:15:02,835 INFO    MainThread:1754682 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-31 22:15:02,835 INFO    MainThread:1754682 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-31 22:15:02,836 INFO    MainThread:1754682 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-31 22:15:02,837 INFO    MainThread:1754682 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-31 22:15:02,837 INFO    MainThread:1754682 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/gwnrzhmx/tb, True
+2024-12-31 22:15:13,400 INFO    MainThread:1754682 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-31 22:19:22,339 WARNING MsgRouterThr:1754682 [router.py:message_loop():77] message_loop has been closed
diff --git a/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/run-gwnrzhmx.wandb b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/run-gwnrzhmx.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..6d813ab60416458672fd24ee70bda254d30edff3
Binary files /dev/null and b/gwnrzhmx/wandb/run-20241231_221502-gwnrzhmx/run-gwnrzhmx.wandb differ
diff --git a/hab6e0c2/episode_rosbags/episode_0_2024-12-31-22-41-43.bag b/hab6e0c2/episode_rosbags/episode_0_2024-12-31-22-41-43.bag
new file mode 100644
index 0000000000000000000000000000000000000000..f78962f21fc931016d04ee88e98eaf054ebb5e81
--- /dev/null
+++ b/hab6e0c2/episode_rosbags/episode_0_2024-12-31-22-41-43.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ecf4c9df023017e8579f76042423262fb16db82747dcf3244f22f8b5a2e82d5
+size 1410495671
diff --git a/hab6e0c2/episode_rosbags/episode_1_2024-12-31-22-42-42.bag b/hab6e0c2/episode_rosbags/episode_1_2024-12-31-22-42-42.bag
new file mode 100644
index 0000000000000000000000000000000000000000..d34a0008ef23d70f98295b3ef06018a2e57265e4
--- /dev/null
+++ b/hab6e0c2/episode_rosbags/episode_1_2024-12-31-22-42-42.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f872746210e8669926b97bb161accfecec0fd01bf2a48dac9cf5734c11ff8ad
+size 1411280931
diff --git a/hab6e0c2/episode_rosbags/episode_2_2024-12-31-22-43-30.bag b/hab6e0c2/episode_rosbags/episode_2_2024-12-31-22-43-30.bag
new file mode 100644
index 0000000000000000000000000000000000000000..88ca21866cea7996a03e58a68a04694bba1781c0
--- /dev/null
+++ b/hab6e0c2/episode_rosbags/episode_2_2024-12-31-22-43-30.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd545b8ec8b53741a496eb034df95e5d69e20164483b873849da51797e07da88
+size 1412914975
diff --git a/hab6e0c2/episode_rosbags/episode_3_2024-12-31-22-44-16.bag b/hab6e0c2/episode_rosbags/episode_3_2024-12-31-22-44-16.bag
new file mode 100644
index 0000000000000000000000000000000000000000..b628cc83e9c96f949558f00e43e75c2c3cde468f
--- /dev/null
+++ b/hab6e0c2/episode_rosbags/episode_3_2024-12-31-22-44-16.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ad41e239a03f8be019aa7511d973a9df27100d03e569f734203ee1b088e15a3
+size 1412475674
diff --git a/hab6e0c2/episode_rosbags/episode_4_2024-12-31-22-45-00.bag b/hab6e0c2/episode_rosbags/episode_4_2024-12-31-22-45-00.bag
new file mode 100644
index 0000000000000000000000000000000000000000..8cae08815a34f017fdfed791bf5d707d446ec9e5
--- /dev/null
+++ b/hab6e0c2/episode_rosbags/episode_4_2024-12-31-22-45-00.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69ddfc3a9717b01734421bbde5c0d9b71ef3b7b693c722dfd22ef136fd6e0660
+size 1408376629
diff --git a/mi8hllsu/.hydra/config.yaml b/mi8hllsu/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..bd6afe16b08d9c5459bedf26d3f31e68b999cbbe
--- /dev/null
+++ b/mi8hllsu/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1007_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/mi8hllsu/.hydra/hydra.yaml b/mi8hllsu/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c32b83f74683183185acefb468e184a418dd7636
--- /dev/null
+++ b/mi8hllsu/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/mi8hllsu
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/mi8hllsu/.hydra/overrides.yaml b/mi8hllsu/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/mi8hllsu/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/mi8hllsu/eval_policy.log b/mi8hllsu/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..eb30de575a349b3d77d79d06336bfd3bc3cab838
--- /dev/null
+++ b/mi8hllsu/eval_policy.log
@@ -0,0 +1,15 @@
+[2024-12-31 22:14:52,304][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-31 22:14:52,307][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-31 22:14:57,014][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+
diff --git a/qo464509/episode_rosbags/episode_1_2025-01-01-10-53-45.bag b/qo464509/episode_rosbags/episode_1_2025-01-01-10-53-45.bag
new file mode 100644
index 0000000000000000000000000000000000000000..f44f95331972cfcc5fd81b88761493f552783c08
--- /dev/null
+++ b/qo464509/episode_rosbags/episode_1_2025-01-01-10-53-45.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69f38ce2118a2eb012fa06ada0178b830f2b8aa96aa7c24f5b4529755afb0005
+size 1412866893
diff --git a/qo464509/episode_rosbags/episode_3_2025-01-01-10-55-17.bag b/qo464509/episode_rosbags/episode_3_2025-01-01-10-55-17.bag
new file mode 100644
index 0000000000000000000000000000000000000000..a752156b0c7ab25cbcf24e1193b01a92c8f8297f
--- /dev/null
+++ b/qo464509/episode_rosbags/episode_3_2025-01-01-10-55-17.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9265b5a0c232f52aa4f0bf8eeca0fc169da16f7c64c8e5d16b4bdaa389977ea
+size 1412588747
diff --git a/snapshot_12000.pt b/snapshot_12000.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7c05e22fa12de61e8b25182d4e3bfa1280f80997
--- /dev/null
+++ b/snapshot_12000.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f7a95220fb56aefb23b17d5c9303e2c904a14e8e5db30c3059357b4d8892690
+size 910102870