diff --git a/.gitattributes b/.gitattributes
index b60f99843a9dc857cdbc82ebebc05275ca3ffe59..9d92971654df953496b576d1b37ba496607ef9d7 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -78,3 +78,33 @@ z0y7vp39/episode_rosbags/episode_1_2024-12-31-21-29-58.bag filter=lfs diff=lfs m
 z0y7vp39/episode_rosbags/episode_2_2024-12-31-21-30-36.bag filter=lfs diff=lfs merge=lfs -text
 z0y7vp39/episode_rosbags/episode_0_2024-12-31-21-29-21.bag filter=lfs diff=lfs merge=lfs -text
 6rp1ws1k/episode_rosbags/episode_3_2024-12-31-22-31-02.bag filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/episode_rosbags/episode_4_2024-12-31-22-31-55.bag filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4 filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4 filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4 filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4 filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4 filter=lfs diff=lfs merge=lfs -text
+lydwde6h/episode_rosbags/episode_2_2025-01-01-11-18-39.bag filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+lydwde6h/episode_rosbags/episode_4_2025-01-01-11-20-32.bag filter=lfs diff=lfs merge=lfs -text
+lydwde6h/episode_rosbags/episode_3_2025-01-01-11-19-37.bag filter=lfs diff=lfs merge=lfs -text
+6rp1ws1k/episode_rosbags/episode_2_2024-12-31-22-30-13.bag filter=lfs diff=lfs merge=lfs -text
+7j9cp4xk/episode_rosbags/episode_2_2025-01-01-00-10-52.bag filter=lfs diff=lfs merge=lfs -text
+c7zbixne/wandb/run-20241231_213503-c7zbixne/run-c7zbixne.wandb filter=lfs diff=lfs merge=lfs -text
+7j9cp4xk/episode_rosbags/episode_3_2025-01-01-00-11-48.bag filter=lfs diff=lfs merge=lfs -text
+c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4 filter=lfs diff=lfs merge=lfs -text
+c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4 filter=lfs diff=lfs merge=lfs -text
+c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4 filter=lfs diff=lfs merge=lfs -text
+c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4 filter=lfs diff=lfs merge=lfs -text
+c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4 filter=lfs diff=lfs merge=lfs -text
+c7zbixne/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+c7zbixne/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+c7zbixne/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+c7zbixne/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+c7zbixne/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+c7zbixne/episode_rosbags/episode_1_2024-12-31-21-36-35.bag filter=lfs diff=lfs merge=lfs -text
+c7zbixne/episode_rosbags/episode_2_2024-12-31-21-37-20.bag filter=lfs diff=lfs merge=lfs -text
diff --git a/5r8mx4bu/episode_rosbags/aligned_depth_to_color_K.npy b/5r8mx4bu/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/5r8mx4bu/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/5r8mx4bu/episode_rosbags/cam_tf_world.npy b/5r8mx4bu/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/5r8mx4bu/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/5r8mx4bu/episode_rosbags/depth_K.npy b/5r8mx4bu/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/5r8mx4bu/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/5r8mx4bu/eval_robot.log b/5r8mx4bu/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..f8cf5e99f1834b537a4dcc09c98c3abe61bff95a
--- /dev/null
+++ b/5r8mx4bu/eval_robot.log
@@ -0,0 +1,12 @@
+[2024-12-31 23:59:47,575][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-31 23:59:47,578][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/6rp1ws1k/.hydra/config.yaml b/6rp1ws1k/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90
--- /dev/null
+++ b/6rp1ws1k/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1006_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/6rp1ws1k/.hydra/hydra.yaml b/6rp1ws1k/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c9930c3d3279981e247a79d38132c379dd2a38b4
--- /dev/null
+++ b/6rp1ws1k/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/6rp1ws1k/.hydra/overrides.yaml b/6rp1ws1k/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/6rp1ws1k/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/6rp1ws1k/episode_rosbags/aligned_depth_to_color_K.npy b/6rp1ws1k/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/6rp1ws1k/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/6rp1ws1k/episode_rosbags/color_K.npy b/6rp1ws1k/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/6rp1ws1k/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/6rp1ws1k/episode_rosbags/episode_2_2024-12-31-22-30-13.bag b/6rp1ws1k/episode_rosbags/episode_2_2024-12-31-22-30-13.bag
new file mode 100644
index 0000000000000000000000000000000000000000..313f3c54e1402b309ebe442215b7f6301dde8eeb
--- /dev/null
+++ b/6rp1ws1k/episode_rosbags/episode_2_2024-12-31-22-30-13.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:876cc1267a39cf6585330f8ae9d1202215be3f6c817570822287ddc77992c01e
+size 2613073524
diff --git a/6rp1ws1k/episode_rosbags/episode_4_2024-12-31-22-31-55.bag b/6rp1ws1k/episode_rosbags/episode_4_2024-12-31-22-31-55.bag
new file mode 100644
index 0000000000000000000000000000000000000000..561fb9a99fb35b759b1fff9ad6af6a7e2448c9f0
--- /dev/null
+++ b/6rp1ws1k/episode_rosbags/episode_4_2024-12-31-22-31-55.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:189278a42199e74a2750176ef337d78769bcc8f70b6453ba9a628b0a302af013
+size 2608902846
diff --git a/6rp1ws1k/eval_video/0_eval.mp4 b/6rp1ws1k/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4f8f4dc4a603deee0b55059157985a36e051785b
--- /dev/null
+++ b/6rp1ws1k/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:037a1e72b853dceefbcd9b1169ecaabb8fffadc0dbf620bc7d2135f44e9498b5
+size 1448432
diff --git a/6rp1ws1k/eval_video/1_eval.mp4 b/6rp1ws1k/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5377bf4a0a3f038de35533b7d3297a3b11eb92bb
--- /dev/null
+++ b/6rp1ws1k/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cd7d5b3a1a16852a937778c1e1a539693a6343a7f43a0e2429da7ff2ef1ed25
+size 1456245
diff --git a/6rp1ws1k/eval_video/2_eval.mp4 b/6rp1ws1k/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..2db4944f8ad9ff9252212172bbd34e18ea7c2985
--- /dev/null
+++ b/6rp1ws1k/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c119cb8d8d9251fce33ac9d5947151bac092d2791d5ae11953edf7dd7192308
+size 1445511
diff --git a/6rp1ws1k/eval_video/3_eval.mp4 b/6rp1ws1k/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..fc4573c86f446db6fe03cb3e80977f7d9c618a66
--- /dev/null
+++ b/6rp1ws1k/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ed43f7879c76b869ca4774ddb3ad21ed2b125c8f32d320c3c96b729f38949ab
+size 1456536
diff --git a/6rp1ws1k/eval_video/4_eval.mp4 b/6rp1ws1k/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..356b253899d30aa751db432c15d0ee323abdd239
--- /dev/null
+++ b/6rp1ws1k/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9d47729d90bd1ce9981d28056487f31708499f38a875bb50151d73f5e221462
+size 1553312
diff --git a/6rp1ws1k/tb/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0 b/6rp1ws1k/tb/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
new file mode 100644
index 0000000000000000000000000000000000000000..ae517c24bf719e1d09304653ce1d84993c0f68db
--- /dev/null
+++ b/6rp1ws1k/tb/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7f8d113eedbf160c9871742ae5a06f52464fa09c43d4215aaa5cd4a20cfea01
+size 6539
diff --git a/6rp1ws1k/wandb/debug-internal.log b/6rp1ws1k/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..bfc582bde148a480f111366298e1e7151e341ffa
--- /dev/null
+++ b/6rp1ws1k/wandb/debug-internal.log
@@ -0,0 +1,818 @@
+2024-12-31 22:27:52,843 INFO    StreamThr :1757972 [internal.py:wandb_internal():85] W&B internal server running at pid: 1757972, started at: 2024-12-31 22:27:52.843051
+2024-12-31 22:27:52,844 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status
+2024-12-31 22:27:52,856 INFO    WriterThread:1757972 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/run-6rp1ws1k.wandb
+2024-12-31 22:27:52,857 DEBUG   SenderThread:1757972 [sender.py:send():379] send: header
+2024-12-31 22:27:52,864 DEBUG   SenderThread:1757972 [sender.py:send():379] send: run
+2024-12-31 22:27:53,094 INFO    SenderThread:1757972 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files
+2024-12-31 22:27:53,094 INFO    SenderThread:1757972 [sender.py:_start_run_threads():1188] run started: 6rp1ws1k with start time 1735702072.844013
+2024-12-31 22:27:53,113 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: check_version
+2024-12-31 22:27:53,114 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: check_version
+2024-12-31 22:27:53,184 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: run_start
+2024-12-31 22:27:53,189 DEBUG   HandlerThread:1757972 [system_info.py:__init__():26] System info init
+2024-12-31 22:27:53,189 DEBUG   HandlerThread:1757972 [system_info.py:__init__():41] System info init done
+2024-12-31 22:27:53,189 INFO    HandlerThread:1757972 [system_monitor.py:start():194] Starting system monitor
+2024-12-31 22:27:53,189 INFO    SystemMonitor:1757972 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-31 22:27:53,190 INFO    HandlerThread:1757972 [system_monitor.py:probe():214] Collecting system info
+2024-12-31 22:27:53,190 INFO    SystemMonitor:1757972 [interfaces.py:start():188] Started cpu monitoring
+2024-12-31 22:27:53,191 INFO    SystemMonitor:1757972 [interfaces.py:start():188] Started disk monitoring
+2024-12-31 22:27:53,191 INFO    SystemMonitor:1757972 [interfaces.py:start():188] Started gpu monitoring
+2024-12-31 22:27:53,192 INFO    SystemMonitor:1757972 [interfaces.py:start():188] Started memory monitoring
+2024-12-31 22:27:53,192 INFO    SystemMonitor:1757972 [interfaces.py:start():188] Started network monitoring
+2024-12-31 22:27:53,203 DEBUG   HandlerThread:1757972 [system_info.py:probe():152] Probing system
+2024-12-31 22:27:53,204 DEBUG   HandlerThread:1757972 [system_info.py:_probe_git():137] Probing git
+2024-12-31 22:27:53,209 DEBUG   HandlerThread:1757972 [system_info.py:_probe_git():145] Probing git done
+2024-12-31 22:27:53,209 DEBUG   HandlerThread:1757972 [system_info.py:probe():200] Probing system done
+2024-12-31 22:27:53,209 DEBUG   HandlerThread:1757972 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T03:27:53.203312', 'startedAt': '2025-01-01T03:27:52.830373', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3934.60025, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 4496.753, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 548.2166061401367}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-31 22:27:53,209 INFO    HandlerThread:1757972 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-31 22:27:53,209 INFO    HandlerThread:1757972 [system_monitor.py:probe():227] Publishing system info
+2024-12-31 22:27:53,209 DEBUG   HandlerThread:1757972 [system_info.py:_save_code():44] Saving code
+2024-12-31 22:27:53,211 DEBUG   HandlerThread:1757972 [system_info.py:_save_code():67] Saving code done
+2024-12-31 22:27:53,211 DEBUG   HandlerThread:1757972 [system_info.py:_save_patches():84] Saving git patches
+2024-12-31 22:27:53,274 DEBUG   HandlerThread:1757972 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-31 22:27:53,274 INFO    HandlerThread:1757972 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-31 22:27:53,277 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:27:53,277 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-31 22:27:53,278 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-31 22:27:53,278 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-31 22:27:53,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-31 22:27:53,518 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: python_packages
+2024-12-31 22:27:53,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:27:53,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:27:53,543 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/qjzcudv1-wandb-metadata.json
+2024-12-31 22:27:53,557 INFO    wandb-upload_2:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/fljjd6ev-diff.patch
+2024-12-31 22:27:53,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:53,561 INFO    HandlerThread:1757972 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/tb"
+  save: true
+}
+control {
+  relay_id: "51498112503d47c680ad8846065484a9"
+}
+_info {
+  stream_id: "6rp1ws1k"
+}
+
+2024-12-31 22:27:53,561 DEBUG   HandlerThread:1757972 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-31 22:27:53,572 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/liwjjhlk-code/FISH/eval_robot.py
+2024-12-31 22:27:53,665 DEBUG   SenderThread:1757972 [sender.py:send():379] send: telemetry
+2024-12-31 22:27:53,665 DEBUG   SenderThread:1757972 [sender.py:send():379] send: tbrecord
+2024-12-31 22:27:53,666 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:27:53,666 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0 with policy live
+2024-12-31 22:27:53,851 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/5wmw6s8p-events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code/FISH/eval_robot.py
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/diff.patch
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/requirements.txt
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-metadata.json
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:27:54,097 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code/FISH
+2024-12-31 22:27:54,097 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code
+2024-12-31 22:27:54,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:55,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:56,095 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:27:56,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:57,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:58,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:27:58,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:58,591 DEBUG   SenderThread:1757972 [sender.py:send():379] send: config
+2024-12-31 22:27:59,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:00,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:00,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:01,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:02,097 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:02,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:03,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:03,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:04,097 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:04,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:05,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:06,098 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:06,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:07,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:08,099 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:08,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:28:08,518 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:28:08,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:08,669 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:09,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:10,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:11,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:12,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:13,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:14,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:14,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:15,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:16,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:17,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:18,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:19,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:19,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:20,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:21,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:22,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:23,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:28:23,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:28:23,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:24,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:24,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:25,103 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml
+2024-12-31 22:28:25,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:26,104 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:26,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:27,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:28,105 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:28,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:29,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:30,035 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:30,105 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:30,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:31,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:32,106 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:32,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:33,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:34,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:35,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:35,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:36,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:37,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:38,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:28:38,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:28:38,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:39,525 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:40,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:40,665 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:41,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:42,108 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:42,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:43,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:44,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:45,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:46,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:46,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:47,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:48,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:49,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:50,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:51,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:51,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:52,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:53,192 DEBUG   SystemMonitor:1757972 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-31 22:28:53,193 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:28:53,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:28:53,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:28:53,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:54,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:55,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:56,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:56,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:57,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:58,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:59,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:00,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:01,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:02,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:02,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:03,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:04,481 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:29:04,481 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4 with policy now
+2024-12-31 22:29:04,482 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:29:04,483 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:29:04,483 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:29:04,486 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:29:04,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:04,749 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/tqv3t9x2-media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4
+2024-12-31 22:29:05,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:29:05,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4
+2024-12-31 22:29:05,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media
+2024-12-31 22:29:05,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval
+2024-12-31 22:29:05,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos
+2024-12-31 22:29:05,398 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/cg2gg81j-media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4
+2024-12-31 22:29:05,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:06,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:06,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:07,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:08,000 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:08,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:29:08,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:29:08,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:09,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:10,116 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:10,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:11,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:12,117 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:12,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:13,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:13,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:14,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:15,118 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:29:15,373 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/m4e9d6ct-events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:29:15,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:16,118 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:16,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:17,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:18,119 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:18,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:18,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:19,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:20,120 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:20,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:21,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:22,121 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:22,531 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:23,193 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:29:23,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:29:23,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:29:23,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:23,590 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:24,531 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:25,532 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:26,122 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:26,531 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:27,531 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:28,533 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:29,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:29,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:30,534 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:31,534 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:32,535 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:33,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:34,538 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:35,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:35,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:36,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:37,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:38,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:29:38,521 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:29:38,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:39,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:40,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:40,593 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:41,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:42,537 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:43,537 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:44,538 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:45,538 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:46,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:46,543 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:47,542 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:48,544 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:49,544 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:50,543 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:51,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:51,543 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:52,544 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:53,194 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:29:53,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:29:53,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:29:53,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:54,543 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:55,258 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:29:55,258 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4 with policy now
+2024-12-31 22:29:55,260 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:29:55,260 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:29:55,260 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:29:55,281 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:29:55,523 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/utmzm97k-media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4
+2024-12-31 22:29:55,544 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:56,132 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:29:56,132 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:56,132 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4
+2024-12-31 22:29:56,132 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval
+2024-12-31 22:29:56,429 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/eu7mhhbv-media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4
+2024-12-31 22:29:56,546 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:57,282 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:57,545 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:58,551 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:59,551 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:00,133 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:00,546 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:01,569 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:02,548 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:02,570 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:03,546 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:04,134 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:04,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:05,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:06,549 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:07,549 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:08,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:30:08,524 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:30:08,569 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:08,592 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:09,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:10,136 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:30:10,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:10,996 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:30:10,996 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:30:10,996 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:30:11,136 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:30:11,136 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:30:11,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:12,137 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:12,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:13,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:13,904 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:14,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:15,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:16,138 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:16,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:17,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:18,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:19,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:19,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:20,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:21,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:22,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:23,194 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:30:23,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:30:23,520 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:30:23,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:24,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:24,627 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:25,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:26,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:27,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:28,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:29,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:30,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:30,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:31,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:32,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:33,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:34,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:35,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:35,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:36,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:37,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:38,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:30:38,520 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:30:38,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:39,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:40,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:40,596 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:41,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:42,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:43,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:44,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:45,556 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:46,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:46,556 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:47,558 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:48,557 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:49,560 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:50,062 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:30:50,062 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4 with policy now
+2024-12-31 22:30:50,063 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:30:50,063 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:30:50,063 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:30:50,065 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:30:50,149 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:30:50,149 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:50,149 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4
+2024-12-31 22:30:50,150 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval
+2024-12-31 22:30:50,360 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/y9cwp5qi-media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4
+2024-12-31 22:30:50,558 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:50,645 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/1cr9sa4t-media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4
+2024-12-31 22:30:51,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:52,101 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:52,150 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:52,568 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:53,195 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:30:53,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:30:53,522 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:30:53,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:54,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:55,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:56,151 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:56,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:57,579 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:57,617 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:58,152 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:58,566 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:59,152 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:30:59,428 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/gogcd3wz-events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:30:59,574 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:59,839 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:30:59,839 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:30:59,840 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:31:00,153 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:31:00,153 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:00,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:01,581 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:02,153 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:02,573 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:02,717 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:03,570 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:04,154 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:04,570 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:05,570 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:06,155 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:06,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:07,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:07,724 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:08,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:31:08,520 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:31:08,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:09,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:10,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:11,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:12,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:13,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:13,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:14,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:15,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:16,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:17,575 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:18,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:18,573 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:19,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:20,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:21,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:22,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:23,195 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:31:23,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:31:23,521 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:31:23,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:23,669 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:24,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:25,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:26,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:27,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:28,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:29,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:29,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:30,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:31,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:32,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:33,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:34,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:34,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:35,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:36,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:37,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:38,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:31:38,521 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:31:38,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:39,393 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:31:39,393 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4 with policy now
+2024-12-31 22:31:39,398 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:31:39,398 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:31:39,398 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:31:39,413 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:31:39,584 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:39,664 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/x7305ls9-media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4
+2024-12-31 22:31:40,166 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:31:40,166 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:40,166 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4
+2024-12-31 22:31:40,174 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval
+2024-12-31 22:31:40,450 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:40,496 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/2zh2ping-media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4
+2024-12-31 22:31:40,583 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:41,587 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:42,167 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:42,587 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:43,586 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:44,167 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:44,589 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:45,547 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:45,592 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:46,592 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:47,587 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:48,169 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:48,589 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:49,591 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:50,593 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:51,260 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:51,597 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:51,905 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:31:51,905 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:31:51,905 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:31:52,170 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:31:52,170 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:31:52,170 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:52,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:53,196 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:31:53,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:31:53,524 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:31:53,600 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:54,171 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:54,611 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:55,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:56,171 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:56,608 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:57,185 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:57,604 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:58,172 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:58,604 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:59,604 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:00,173 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:00,605 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:01,605 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:02,605 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:02,805 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:03,607 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:04,174 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:04,607 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:05,607 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:06,175 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:06,607 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:07,608 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:08,023 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:08,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:32:08,523 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:32:08,608 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:09,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:10,177 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:10,608 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:11,608 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:12,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:13,533 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:13,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:14,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:15,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:16,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:17,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:18,179 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:18,534 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:18,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:19,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:20,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:21,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:22,611 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:23,196 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:32:23,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:32:23,523 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:32:23,611 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:23,657 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:24,611 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:25,611 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:26,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:27,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:28,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:29,533 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:29,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:30,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:31,613 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:32,613 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:33,626 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:34,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:34,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
+2024-12-31 22:32:34,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval
+2024-12-31 22:32:34,205 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:32:34,205 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4 with policy now
+2024-12-31 22:32:34,210 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:32:34,211 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:32:34,211 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:34,211 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:34,494 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/90n6cfmi-media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
+2024-12-31 22:32:34,615 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:35,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
+2024-12-31 22:32:35,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:35,253 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:35,523 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/816v1cg4-media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
+2024-12-31 22:32:35,629 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:36,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:36,636 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:37,627 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:38,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:32:38,536 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:32:38,647 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:39,640 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:40,187 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:40,622 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:40,634 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:41,633 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:42,187 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:42,644 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:43,639 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:44,188 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:32:44,436 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/0ywzqbxo-events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:32:44,615 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:32:44,615 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:32:44,635 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:44,774 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:32:45,104 INFO    wandb-upload_0:1757972 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpco79vl2s
+2024-12-31 22:32:45,167 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:32:45,181 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval
+2024-12-31 22:32:45,517 INFO    SenderThread:1757972 [sender.py:send_request_log_artifact():1525] logged artifact run-6rp1ws1k-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE2MzY4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MzI2MQ==', 'latestArtifact': None}}
+2024-12-31 22:32:45,517 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:32:45,517 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2024-12-31 22:32:45,517 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:32:45,517 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,518 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,518 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:32:45,518 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,522 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,522 DEBUG   SenderThread:1757972 [sender.py:send():379] send: telemetry
+2024-12-31 22:32:45,522 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,522 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,523 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,523 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,523 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,523 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,523 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,524 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,524 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,524 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,524 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,525 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,525 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,525 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,525 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,525 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,526 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,526 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,526 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,526 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,527 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,527 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,527 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:32:45,632 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:45,707 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/sl2foi94-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:32:45,718 INFO    wandb-upload_2:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/ahec4sm7-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json
+2024-12-31 22:32:45,884 INFO    wandb-upload_0:1757972 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplopob7gl
+2024-12-31 22:32:45,956 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/fwu04w86-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:32:46,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:46,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:46,286 INFO    SenderThread:1757972 [sender.py:send_request_log_artifact():1525] logged artifact run-6rp1ws1k-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE2MzY5NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MzI2Mg==', 'latestArtifact': None}}
+2024-12-31 22:32:46,289 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:46,403 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:32:46,403 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json with policy now
+2024-12-31 22:32:46,403 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:32:46,403 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:46,404 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:46,633 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:46,670 INFO    wandb-upload_2:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/970y0upf-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json
+2024-12-31 22:32:47,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:47,190 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml
+2024-12-31 22:32:47,632 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:48,190 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:48,633 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:49,633 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:49,909 DEBUG   SenderThread:1757972 [sender.py:send():379] send: exit
+2024-12-31 22:32:49,909 INFO    SenderThread:1757972 [sender.py:send_exit():586] handling exit code: 0
+2024-12-31 22:32:49,910 INFO    SenderThread:1757972 [sender.py:send_exit():588] handling runtime: 296
+2024-12-31 22:32:49,910 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:49,910 INFO    SenderThread:1757972 [sender.py:send_exit():594] send defer
+2024-12-31 22:32:49,910 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:49,910 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-31 22:32:49,910 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:49,910 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-31 22:32:49,910 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 1
+2024-12-31 22:32:49,911 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:49,911 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-31 22:32:49,911 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:49,911 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-31 22:32:49,911 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 2
+2024-12-31 22:32:49,911 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:49,911 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-31 22:32:49,911 INFO    HandlerThread:1757972 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-31 22:32:49,911 DEBUG   SystemMonitor:1757972 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-31 22:32:49,911 DEBUG   SystemMonitor:1757972 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-31 22:32:49,911 INFO    HandlerThread:1757972 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-31 22:32:49,912 INFO    HandlerThread:1757972 [interfaces.py:finish():200] Joined disk monitor
+2024-12-31 22:32:49,981 INFO    HandlerThread:1757972 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-31 22:32:49,981 INFO    HandlerThread:1757972 [interfaces.py:finish():200] Joined memory monitor
+2024-12-31 22:32:49,982 INFO    HandlerThread:1757972 [interfaces.py:finish():200] Joined network monitor
+2024-12-31 22:32:49,982 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:49,982 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-31 22:32:49,982 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 3
+2024-12-31 22:32:49,982 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:32:49,982 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:49,982 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-31 22:32:49,983 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:49,983 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-31 22:32:49,983 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 4
+2024-12-31 22:32:49,983 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:49,983 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-31 22:32:50,191 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:52,192 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:56,978 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:32:56,978 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:56,978 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-31 22:32:56,979 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:56,979 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-31 22:32:56,979 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 5
+2024-12-31 22:32:56,979 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:32:56,979 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:32:56,979 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:56,979 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:56,979 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-31 22:32:56,980 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:56,980 DEBUG   SenderThread:1757972 [sender.py:send():379] send: summary
+2024-12-31 22:32:56,984 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:56,984 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:56,984 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-31 22:32:56,984 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:56,984 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 6
+2024-12-31 22:32:56,984 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:56,984 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-31 22:32:56,985 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:56,985 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-31 22:32:56,985 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 7
+2024-12-31 22:32:56,985 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:56,985 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:56,985 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-31 22:32:56,985 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:56,985 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-31 22:32:57,194 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:57,911 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:32:57,989 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 8
+2024-12-31 22:32:57,989 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:32:57,989 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:57,989 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-31 22:32:57,990 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:57,990 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-31 22:32:57,990 INFO    SenderThread:1757972 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-31 22:32:57,992 INFO    SenderThread:1757972 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-31 22:32:57,994 INFO    SenderThread:1757972 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-31 22:32:58,011 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 9
+2024-12-31 22:32:58,011 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:58,011 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-31 22:32:58,012 DEBUG   SenderThread:1757972 [sender.py:send():379] send: artifact
+2024-12-31 22:32:58,195 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:58,410 INFO    wandb-upload_2:1757972 [upload_job.py:push():85] Skipped uploading /tmp/tmpqrps9wt9/wandb-job.json
+2024-12-31 22:32:58,410 INFO    wandb-upload_0:1757972 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmmq90y7h
+2024-12-31 22:32:58,530 INFO    wandb-upload_1:1757972 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2s46j5eq
+2024-12-31 22:32:58,912 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:32:59,161 INFO    SenderThread:1757972 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE2MzgxOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE1OTY3NQ==', 'versionIndex': 396}}}
+2024-12-31 22:32:59,161 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,161 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-31 22:32:59,161 INFO    SenderThread:1757972 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-31 22:32:59,196 INFO    SenderThread:1757972 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files
+2024-12-31 22:32:59,196 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml config.yaml
+2024-12-31 22:32:59,196 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json wandb-summary.json
+2024-12-31 22:32:59,197 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0 events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/requirements.txt requirements.txt
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-metadata.json wandb-metadata.json
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log output.log
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/diff.patch diff.patch
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4 media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4 media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4
+2024-12-31 22:32:59,201 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4 media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
+2024-12-31 22:32:59,201 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4 media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4
+2024-12-31 22:32:59,201 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4 media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4
+2024-12-31 22:32:59,204 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json
+2024-12-31 22:32:59,204 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:32:59,204 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 10
+2024-12-31 22:32:59,204 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:32:59,204 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:59,204 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-31 22:32:59,206 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,206 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-31 22:32:59,206 INFO    SenderThread:1757972 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:32:59,379 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:59,399 INFO    wandb-upload_2:1757972 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml
+2024-12-31 22:32:59,432 INFO    wandb-upload_4:1757972 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:59,449 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/requirements.txt
+2024-12-31 22:32:59,496 INFO    wandb-upload_3:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/uo92ha99-events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:32:59,696 INFO    Thread-11 (_thread_body):1757972 [sender.py:transition_state():614] send defer: 11
+2024-12-31 22:32:59,696 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:59,696 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-31 22:32:59,697 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,697 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-31 22:32:59,697 INFO    SenderThread:1757972 [file_pusher.py:join():175] waiting for file pusher
+2024-12-31 22:32:59,698 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 12
+2024-12-31 22:32:59,698 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:59,698 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-31 22:32:59,698 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,698 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-31 22:32:59,698 INFO    SenderThread:1757972 [file_stream.py:finish():601] file stream finish called
+2024-12-31 22:32:59,895 INFO    SenderThread:1757972 [file_stream.py:finish():605] file stream finish is done
+2024-12-31 22:32:59,895 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 13
+2024-12-31 22:32:59,895 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:59,895 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-31 22:32:59,895 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,895 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-31 22:32:59,895 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 14
+2024-12-31 22:32:59,896 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:59,896 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-31 22:32:59,896 DEBUG   SenderThread:1757972 [sender.py:send():379] send: final
+2024-12-31 22:32:59,896 DEBUG   SenderThread:1757972 [sender.py:send():379] send: footer
+2024-12-31 22:32:59,896 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,896 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-31 22:32:59,896 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:32:59,897 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:32:59,897 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: server_info
+2024-12-31 22:32:59,897 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:32:59,897 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-31 22:32:59,898 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-31 22:32:59,899 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:59,899 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: server_info
+2024-12-31 22:32:59,949 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:32:59,950 INFO    MainThread:1757972 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-31 22:32:59,950 INFO    MainThread:1757972 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-31 22:32:59,950 INFO    MainThread:1757972 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-31 22:32:59,951 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-31 22:32:59,951 INFO    HandlerThread:1757972 [handler.py:finish():882] shutting down handler
+2024-12-31 22:33:00,899 INFO    WriterThread:1757972 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/run-6rp1ws1k.wandb
+2024-12-31 22:33:00,950 INFO    SenderThread:1757972 [sender.py:finish():1615] shutting down sender
+2024-12-31 22:33:00,950 INFO    SenderThread:1757972 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:33:00,950 INFO    SenderThread:1757972 [file_pusher.py:join():175] waiting for file pusher
diff --git a/6rp1ws1k/wandb/debug.log b/6rp1ws1k/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..b9c1f72b10620f06ceb0d5087e483ba6d43c4fde
--- /dev/null
+++ b/6rp1ws1k/wandb/debug.log
@@ -0,0 +1,31 @@
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Configure stats pid to 1757775
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/settings
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/logs/debug.log
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/logs/debug-internal.log
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:init():569] calling init triggers
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k'}
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:init():619] starting backend
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:init():623] setting up manager
+2024-12-31 22:27:52,842 INFO    MainThread:1757775 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-31 22:27:52,843 INFO    MainThread:1757775 [wandb_init.py:init():631] backend started and connected
+2024-12-31 22:27:52,856 INFO    MainThread:1757775 [wandb_init.py:init():720] updated telemetry
+2024-12-31 22:27:52,862 INFO    MainThread:1757775 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-31 22:27:53,113 INFO    MainThread:1757775 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-31 22:27:53,179 INFO    MainThread:1757775 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-31 22:27:53,179 INFO    MainThread:1757775 [wandb_init.py:init():804] starting run threads in backend
+2024-12-31 22:27:53,518 INFO    MainThread:1757775 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-31 22:27:53,518 INFO    MainThread:1757775 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-31 22:27:53,518 INFO    MainThread:1757775 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-31 22:27:53,518 INFO    MainThread:1757775 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-31 22:27:53,520 INFO    MainThread:1757775 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-31 22:27:53,520 INFO    MainThread:1757775 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/tb, True
+2024-12-31 22:27:58,590 INFO    MainThread:1757775 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-31 22:33:00,951 WARNING MsgRouterThr:1757775 [router.py:message_loop():77] message_loop has been closed
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code/FISH/eval_robot.py b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..6f971c04f88c3f951a2321c29af50f00efe5b3a1
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					total_num_episodes += filtered_run.summary_metrics['episode']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7c19f530e89572009ae902638357cb97816851cf
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1006_0'
+wandb_notes:
+  desc: null
+  value: 1006_0_req_1352_0restarted_2
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: true
+use_sdf_maps:
+  desc: null
+  value: true
+use_normals_maps:
+  desc: null
+  value: true
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: true
+        use_sdf_maps: true
+        use_normals_maps: true
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: true
+          use_sdf_maps: true
+          use_normals_maps: true
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1735702072
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: hbm
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/diff.patch b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..2b64c8068a39a4027d3dba47501c672e55e3389a
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/diff.patch
@@ -0,0 +1,69 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 0f6a9ec..01bdd86 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -133,8 +133,17 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
+-wandb_run_id: '1007_0' 
++# wandb_run_id: '1002_0' 
+ 
++# all books, 6/20 demos per book
++# RGBD+mask+act history
++# wandb_run_id: '1007_0' 
++
++# RGBD+mask+contact(all ftrs)+act history
++wandb_run_id: '1006_0'
++
++# RGBD+mask+contact(contact+DTC)+act history
++# wandb_run_id: '1745_0'
+ 
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 21e335c..71a5d1c 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -155,7 +155,12 @@ run_id_list = [
+     # '1022_0',
+     # '1023_0',
+     # '1007_0',
+-    '1007_0'
++    # '1007_0'
++]
++
++run_id_list = [
++    # '1006_0',
++    '1745_0'
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py
+index 4179186..6f971c0 100644
+--- a/FISH/eval_robot.py
++++ b/FISH/eval_robot.py
+@@ -478,8 +478,8 @@ class Workspace:
+ 				"summary_metrics.episode": {"$gte": 5},
+ 				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+ 				"state": "finished",
+-				"config.grasped_obj_name": grasped_obj_name,
+-				"config.left_book_slot": left_book_slot,
++				# "config.grasped_obj_name": grasped_obj_name,
++				# "config.left_book_slot": left_book_slot,
+ 			}
+ 
+ 			api = wandb.Api()
+diff --git a/gym-envs/gym_envs/envs/franka_env.py b/gym-envs/gym_envs/envs/franka_env.py
+index 3302c31..1946eca 100644
+--- a/gym-envs/gym_envs/envs/franka_env.py
++++ b/gym-envs/gym_envs/envs/franka_env.py
+@@ -319,7 +319,7 @@ class FrankaEnv(gym.Env):
+ 
+ 	def mask_obs_callback(self, msg):
+ 		msg = self.possibly_resize_mask_obs(msg)
+-		rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}")
++		# rospy.loginfo(f"received mask_obs with header {msg.header} and grasped_object_mask header {msg.grasped_object_mask.header}")
+ 		self.mask_obs_queue.add(msg.header, msg.grasped_object_mask.header.stamp, msg)  
+ 
+ 	def contact_obs_callback(self, msg):
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]}
\ No newline at end of file
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..bf34235b489a6e5ec51578c4d27b7c5aff91730c
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[10, 20, 0.5, 0.24900326137794287, 0.748354494522015, 0.297579273621989, 0.7263675480628274, 0.32785302404031924, 0.6719539165152919, 0.34575230631336995, 0.6572977080626605]]}
\ No newline at end of file
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4 b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4f8f4dc4a603deee0b55059157985a36e051785b
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:037a1e72b853dceefbcd9b1169ecaabb8fffadc0dbf620bc7d2135f44e9498b5
+size 1448432
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4 b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5377bf4a0a3f038de35533b7d3297a3b11eb92bb
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cd7d5b3a1a16852a937778c1e1a539693a6343a7f43a0e2429da7ff2ef1ed25
+size 1456245
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4 b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..2db4944f8ad9ff9252212172bbd34e18ea7c2985
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c119cb8d8d9251fce33ac9d5947151bac092d2791d5ae11953edf7dd7192308
+size 1445511
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4 b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..fc4573c86f446db6fe03cb3e80977f7d9c618a66
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ed43f7879c76b869ca4774ddb3ad21ed2b125c8f32d320c3c96b729f38949ab
+size 1456536
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4 b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..356b253899d30aa751db432c15d0ee323abdd239
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9d47729d90bd1ce9981d28056487f31708499f38a875bb50151d73f5e221462
+size 1553312
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..2ae0ea77c633b9d808ea534f802257b7f5b13b2d
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
@@ -0,0 +1,267 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1735702078.591438]: resetting environment
+[INFO] [1735702078.599963]: cleared current plan
+[INFO] [1735702078.601671]: moving to home
+[INFO] [1735702079.605309]: reached home
+[INFO] [1735702079.607500]: reset action history
+[INFO] [1735702082.318146]: environment reset
+Starting episode 0
+[INFO] [1735702082.322461]: resetting environment
+[INFO] [1735702082.328240]: cleared current plan
+[INFO] [1735702082.330431]: moving to home
+[INFO] [1735702083.333502]: reached home
+[INFO] [1735702083.334968]: reset action history
+[INFO] [1735702086.044448]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1735702104.639850]: resetting environment
+[INFO] [1735702104.646523]: cleared current plan
+[INFO] [1735702104.647403]: moving to home
+[INFO] [1735702105.649211]: reached home
+[INFO] [1735702105.650554]: reset action history
+[INFO] [1735702108.410538]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/episode_rosbags/episode_0_2024-12-31-22-28-28.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1757775.log
+started roslaunch server http://158.130.50.37:34611/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [1758078]
+process[print_text-2]: started with pid [1758101]
+process[rosbag_record-3]: started with pid [1758102]
+started bagging!
+For topic gripper_width: timestamp difference is 105672735 for nearest: 1735702110019993204 - target: 1735702109914320469 at idx 23
+For topic gripper_width: timestamp difference is 107330490 for nearest: 1735702119619992901 - target: 1735702119512662411 at idx 150
+[INFO] [1735702143.522521]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1735702144.274403]: Stored episode 1.
+[INFO] [1735702144.274760]: Saving video...
+[INFO] [1735702144.470026]: Video saved!
+Starting episode 1
+[INFO] [1735702144.486392]: resetting environment
+[INFO] [1735702144.496883]: cleared current plan
+[INFO] [1735702144.497170]: moving to home
+[INFO] [1735702147.999500]: reached home
+[INFO] [1735702148.000054]: reset action history
+[INFO] [1735702150.711161]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735702154.911076]: resetting environment
+[INFO] [1735702154.918350]: cleared current plan
+[INFO] [1735702154.918884]: moving to home
+[INFO] [1735702155.920307]: reached home
+[INFO] [1735702155.921183]: reset action history
+[INFO] [1735702158.629654]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/episode_rosbags/episode_1_2024-12-31-22-29-18.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1757775.log
+started roslaunch server http://158.130.50.37:45411/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [1758264]
+process[print_text-5]: started with pid [1758265]
+process[rosbag_record-6]: started with pid [1758266]
+started bagging!
+For topic gripper_width: timestamp difference is 106874129 for nearest: 1735702160353326937 - target: 1735702160246452808 at idx 22
+For topic gripper_width: timestamp difference is 110000680 for nearest: 1735702164119992087 - target: 1735702164009991407 at idx 67
+[INFO] [1735702194.132918]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1735702194.942371]: Stored episode 2.
+[INFO] [1735702194.944952]: Saving video...
+[INFO] [1735702195.227766]: Video saved!
+Starting episode 2
+[INFO] [1735702195.262558]: resetting environment
+[INFO] [1735702195.281191]: cleared current plan
+[INFO] [1735702195.281492]: moving to home
+[INFO] [1735702198.784731]: reached home
+[INFO] [1735702198.786719]: reset action history
+[INFO] [1735702201.522178]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735702210.139763]: resetting environment
+[INFO] [1735702210.163846]: cleared current plan
+[INFO] [1735702210.166072]: moving to home
+[INFO] [1735702211.172086]: reached home
+[INFO] [1735702211.174289]: reset action history
+[INFO] [1735702213.901386]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/episode_rosbags/episode_2_2024-12-31-22-30-13.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1757775.log
+started roslaunch server http://158.130.50.37:46069/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [1758420]
+process[print_text-8]: started with pid [1758421]
+process[rosbag_record-9]: started with pid [1758422]
+started bagging!
+[INFO] [1735702248.998231]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1735702249.811900]: Stored episode 3.
+[INFO] [1735702249.820869]: Saving video...
+[INFO] [1735702250.021444]: Video saved!
+Starting episode 3
+[INFO] [1735702250.069079]: resetting environment
+[INFO] [1735702250.100253]: cleared current plan
+[INFO] [1735702250.100757]: moving to home
+[INFO] [1735702253.605627]: reached home
+[INFO] [1735702253.610504]: reset action history
+[INFO] [1735702256.568511]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735702258.909189]: resetting environment
+[INFO] [1735702258.933720]: cleared current plan
+[INFO] [1735702258.934417]: moving to home
+[INFO] [1735702259.936103]: reached home
+[INFO] [1735702259.940036]: reset action history
+[INFO] [1735702262.715548]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/episode_rosbags/episode_3_2024-12-31-22-31-02.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1757775.log
+started roslaunch server http://158.130.50.37:40025/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [1758560]
+process[print_text-11]: started with pid [1758561]
+process[rosbag_record-12]: started with pid [1758562]
+started bagging!
+[INFO] [1735702298.318024]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit[pub_text-10] killing on exit
+[INFO] [1735702299.126822]: Stored episode 4.
+[INFO] [1735702299.129172]: Saving video...
+[INFO] [1735702299.338228]: Video saved!
+Starting episode 4
+[INFO] [1735702299.411059]: resetting environment
+[INFO] [1735702299.446857]: cleared current plan
+[INFO] [1735702299.449656]: moving to home
+[INFO] [1735702302.956845]: reached home
+[INFO] [1735702302.964612]: reset action history
+[INFO] [1735702305.763507]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735702311.304858]: resetting environment
+[INFO] [1735702311.336747]: cleared current plan
+[INFO] [1735702311.340130]: moving to home
+[INFO] [1735702312.349281]: reached home
+[INFO] [1735702312.353902]: reset action history
+[INFO] [1735702315.139855]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/episode_rosbags/episode_4_2024-12-31-22-31-55.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1757775.log
+started roslaunch server http://158.130.50.37:45545/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [1758710]
+process[print_text-14]: started with pid [1758711]
+process[rosbag_record-15]: started with pid [1758712]
+started bagging!
+For topic gripper_width: timestamp difference is 104032970 for nearest: 1735702321753358341 - target: 1735702321649325371 at idx 92
+For topic gripper_width: timestamp difference is 106285421 for nearest: 1735702324619988290 - target: 1735702324513702869 at idx 124
+For topic gripper_width: timestamp difference is 109027966 for nearest: 1735702328019999369 - target: 1735702327910971403 at idx 168
+For topic gripper_width: timestamp difference is 115487854 for nearest: 1735702336019988815 - target: 1735702335904500961 at idx 249
+[INFO] [1735702352.888353]: Storing episode...
+[rosbag_record-15] killing on exit
+[print_text-14] killing on exit[pub_text-13] killing on exit
+[INFO] [1735702353.875075]: Stored episode 5.
+[INFO] [1735702353.882353]: Saving video...
+[INFO] [1735702354.139042]: Video saved!
+[INFO] [1735702354.215133]: resetting environment
+[INFO] [1735702354.252738]: cleared current plan
+[INFO] [1735702354.253158]: moving to home
+[INFO] [1735702357.857682]: reached home
+[INFO] [1735702357.861515]: reset action history
+[INFO] [1735702360.657396]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/requirements.txt b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-metadata.json b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..89602943e3ce9fbb34d81937262b9084c58c5ce9
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-01T03:27:53.203312",
+    "startedAt": "2025-01-01T03:27:52.830373",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3934.60025,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.996,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4496.753,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 548.2166061401367
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..56f8658b21effc63a1f273d991f2c878235ee192
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "037a1e72b853dceefbcd9b1169ecaabb8fffadc0dbf620bc7d2135f44e9498b5", "size": 1448432, "path": "media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4"}, "global_step": 665, "_timestamp": 1735702365.0360737, "_runtime": 292.19206070899963, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "2cd7d5b3a1a16852a937778c1e1a539693a6343a7f43a0e2429da7ff2ef1ed25", "size": 1456245, "path": "media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "6c119cb8d8d9251fce33ac9d5947151bac092d2791d5ae11953edf7dd7192308", "size": 1445511, "path": "media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "0ed43f7879c76b869ca4774ddb3ad21ed2b125c8f32d320c3c96b729f38949ab", "size": 1456536, "path": "media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "e9d47729d90bd1ce9981d28056487f31708499f38a875bb50151d73f5e221462", "size": 1553312, "path": "media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://23uayf5vhexzgw6hv6btoamlqisk6rmrse7rm4rmybhvuk911r6ypcb5zk1hwt2q8t2rqvlngwp6rmw0ebf1jb0xar2rt9ne6skgrl55nl7yax4bc31ykb8u5echuwml/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://4g9nm61baqjpkgu2kq6gl6xdwiosxys2gnf108gx4n37pcju50509x17sgb1z6peer8410o8e3exohwus9z476x2md97alftt9ci7z0p3w1f6vity8qi8c7ku40on9wh:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 10, "total_num_episodes": 20, "total_success_rate": 0.5, "total_success_rate_lb_0.01": 0.24900326137794287, "total_success_rate_ub_0.01": 0.748354494522015, "total_success_rate_lb_0.025": 0.297579273621989, "total_success_rate_ub_0.025": 0.7263675480628274, "total_success_rate_lb_0.05": 0.32785302404031924, "total_success_rate_ub_0.05": 0.6719539165152919, "total_success_rate_lb_0.1": 0.34575230631336995, "total_success_rate_ub_0.1": 0.6572977080626605, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "633be57ab8f5d2a03a6ed414b12b88c46f9388363d35325712a36769135ab23a", "size": 506, "artifact_path": "wandb-client-artifact://ep7yi845vaxpmh30otsypwjsdh06c6v3lppadajgjaf3iyqfeu0quhp3vm567ot8evipf6po6ofa70mp49iqnvcnk3svjjxrcosau1bc1ac2898ktabjf9xcvvwiokaw/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://k25185ghy2may8j1oeytbmkwanl22yz5o3ojlmd9br04f51x3kyscbdwqed2q57k5s9suhinqj2c44pecqoa2e3i3wdt60p602nmcp0h37xlgfj50uqq7cng3baotmvj:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 296}, "eval/total_success_rate": 0.5, "eval/total_success_rate_lb_0.01": 0.24900326132774353, "eval/total_success_rate_ub_0.01": 0.7483544945716858, "eval/total_success_rate_lb_0.025": 0.297579288482666, "eval/total_success_rate_ub_0.025": 0.7263675332069397, "eval/total_success_rate_lb_0.05": 0.3278530240058899, "eval/total_success_rate_ub_0.05": 0.6719539165496826, "eval/total_success_rate_lb_0.1": 0.3457522988319397, "eval/total_success_rate_ub_0.1": 0.6572977304458618}
\ No newline at end of file
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/logs/debug-internal.log b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..bfc582bde148a480f111366298e1e7151e341ffa
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/logs/debug-internal.log
@@ -0,0 +1,818 @@
+2024-12-31 22:27:52,843 INFO    StreamThr :1757972 [internal.py:wandb_internal():85] W&B internal server running at pid: 1757972, started at: 2024-12-31 22:27:52.843051
+2024-12-31 22:27:52,844 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status
+2024-12-31 22:27:52,856 INFO    WriterThread:1757972 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/run-6rp1ws1k.wandb
+2024-12-31 22:27:52,857 DEBUG   SenderThread:1757972 [sender.py:send():379] send: header
+2024-12-31 22:27:52,864 DEBUG   SenderThread:1757972 [sender.py:send():379] send: run
+2024-12-31 22:27:53,094 INFO    SenderThread:1757972 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files
+2024-12-31 22:27:53,094 INFO    SenderThread:1757972 [sender.py:_start_run_threads():1188] run started: 6rp1ws1k with start time 1735702072.844013
+2024-12-31 22:27:53,113 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: check_version
+2024-12-31 22:27:53,114 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: check_version
+2024-12-31 22:27:53,184 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: run_start
+2024-12-31 22:27:53,189 DEBUG   HandlerThread:1757972 [system_info.py:__init__():26] System info init
+2024-12-31 22:27:53,189 DEBUG   HandlerThread:1757972 [system_info.py:__init__():41] System info init done
+2024-12-31 22:27:53,189 INFO    HandlerThread:1757972 [system_monitor.py:start():194] Starting system monitor
+2024-12-31 22:27:53,189 INFO    SystemMonitor:1757972 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-31 22:27:53,190 INFO    HandlerThread:1757972 [system_monitor.py:probe():214] Collecting system info
+2024-12-31 22:27:53,190 INFO    SystemMonitor:1757972 [interfaces.py:start():188] Started cpu monitoring
+2024-12-31 22:27:53,191 INFO    SystemMonitor:1757972 [interfaces.py:start():188] Started disk monitoring
+2024-12-31 22:27:53,191 INFO    SystemMonitor:1757972 [interfaces.py:start():188] Started gpu monitoring
+2024-12-31 22:27:53,192 INFO    SystemMonitor:1757972 [interfaces.py:start():188] Started memory monitoring
+2024-12-31 22:27:53,192 INFO    SystemMonitor:1757972 [interfaces.py:start():188] Started network monitoring
+2024-12-31 22:27:53,203 DEBUG   HandlerThread:1757972 [system_info.py:probe():152] Probing system
+2024-12-31 22:27:53,204 DEBUG   HandlerThread:1757972 [system_info.py:_probe_git():137] Probing git
+2024-12-31 22:27:53,209 DEBUG   HandlerThread:1757972 [system_info.py:_probe_git():145] Probing git done
+2024-12-31 22:27:53,209 DEBUG   HandlerThread:1757972 [system_info.py:probe():200] Probing system done
+2024-12-31 22:27:53,209 DEBUG   HandlerThread:1757972 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T03:27:53.203312', 'startedAt': '2025-01-01T03:27:52.830373', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3934.60025, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 4496.753, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 548.2166061401367}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-31 22:27:53,209 INFO    HandlerThread:1757972 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-31 22:27:53,209 INFO    HandlerThread:1757972 [system_monitor.py:probe():227] Publishing system info
+2024-12-31 22:27:53,209 DEBUG   HandlerThread:1757972 [system_info.py:_save_code():44] Saving code
+2024-12-31 22:27:53,211 DEBUG   HandlerThread:1757972 [system_info.py:_save_code():67] Saving code done
+2024-12-31 22:27:53,211 DEBUG   HandlerThread:1757972 [system_info.py:_save_patches():84] Saving git patches
+2024-12-31 22:27:53,274 DEBUG   HandlerThread:1757972 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-31 22:27:53,274 INFO    HandlerThread:1757972 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-31 22:27:53,277 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:27:53,277 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-31 22:27:53,278 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-31 22:27:53,278 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-31 22:27:53,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-31 22:27:53,518 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: python_packages
+2024-12-31 22:27:53,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:27:53,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:27:53,543 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/qjzcudv1-wandb-metadata.json
+2024-12-31 22:27:53,557 INFO    wandb-upload_2:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/fljjd6ev-diff.patch
+2024-12-31 22:27:53,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:53,561 INFO    HandlerThread:1757972 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/tb"
+  save: true
+}
+control {
+  relay_id: "51498112503d47c680ad8846065484a9"
+}
+_info {
+  stream_id: "6rp1ws1k"
+}
+
+2024-12-31 22:27:53,561 DEBUG   HandlerThread:1757972 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-31 22:27:53,572 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/liwjjhlk-code/FISH/eval_robot.py
+2024-12-31 22:27:53,665 DEBUG   SenderThread:1757972 [sender.py:send():379] send: telemetry
+2024-12-31 22:27:53,665 DEBUG   SenderThread:1757972 [sender.py:send():379] send: tbrecord
+2024-12-31 22:27:53,666 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:27:53,666 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0 with policy live
+2024-12-31 22:27:53,851 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/5wmw6s8p-events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code/FISH/eval_robot.py
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/diff.patch
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/requirements.txt
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-metadata.json
+2024-12-31 22:27:54,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:27:54,097 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code/FISH
+2024-12-31 22:27:54,097 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code
+2024-12-31 22:27:54,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:55,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:56,095 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:27:56,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:57,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:58,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:27:58,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:27:58,591 DEBUG   SenderThread:1757972 [sender.py:send():379] send: config
+2024-12-31 22:27:59,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:00,096 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:00,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:01,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:02,097 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:02,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:03,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:03,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:04,097 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:04,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:05,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:06,098 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:06,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:07,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:08,099 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:08,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:28:08,518 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:28:08,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:08,669 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:09,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:10,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:11,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:12,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:13,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:14,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:14,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:15,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:16,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:17,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:18,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:19,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:19,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:20,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:21,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:22,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:23,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:28:23,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:28:23,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:24,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:24,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:25,103 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml
+2024-12-31 22:28:25,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:26,104 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:26,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:27,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:28,105 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:28,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:29,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:30,035 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:30,105 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:30,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:31,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:32,106 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:32,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:33,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:34,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:35,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:35,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:36,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:37,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:38,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:28:38,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:28:38,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:39,525 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:40,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:40,665 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:41,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:42,108 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:28:42,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:43,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:44,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:45,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:46,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:46,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:47,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:48,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:49,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:50,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:51,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:51,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:52,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:53,192 DEBUG   SystemMonitor:1757972 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-31 22:28:53,193 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:28:53,518 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:28:53,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:28:53,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:54,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:55,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:56,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:56,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:28:57,527 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:58,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:28:59,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:00,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:01,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:02,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:02,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:03,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:04,481 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:29:04,481 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4 with policy now
+2024-12-31 22:29:04,482 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:29:04,483 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:29:04,483 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:29:04,486 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:29:04,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:04,749 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/tqv3t9x2-media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4
+2024-12-31 22:29:05,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:29:05,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4
+2024-12-31 22:29:05,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media
+2024-12-31 22:29:05,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval
+2024-12-31 22:29:05,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos
+2024-12-31 22:29:05,398 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/cg2gg81j-media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4
+2024-12-31 22:29:05,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:06,115 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:06,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:07,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:08,000 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:08,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:29:08,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:29:08,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:09,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:10,116 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:10,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:11,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:12,117 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:12,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:13,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:13,529 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:14,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:15,118 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:29:15,373 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/m4e9d6ct-events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:29:15,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:16,118 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:16,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:17,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:18,119 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:18,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:18,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:19,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:20,120 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:20,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:21,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:22,121 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:22,531 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:23,193 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:29:23,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:29:23,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:29:23,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:23,590 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:24,531 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:25,532 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:26,122 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:26,531 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:27,531 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:28,533 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:29,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:29,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:30,534 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:31,534 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:32,535 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:33,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:34,538 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:35,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:35,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:36,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:37,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:38,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:29:38,521 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:29:38,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:39,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:40,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:40,593 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:41,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:42,537 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:43,537 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:44,538 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:45,538 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:46,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:46,543 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:47,542 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:48,544 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:49,544 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:50,543 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:51,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:51,543 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:52,544 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:53,194 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:29:53,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:29:53,519 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:29:53,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:54,543 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:55,258 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:29:55,258 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4 with policy now
+2024-12-31 22:29:55,260 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:29:55,260 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:29:55,260 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:29:55,281 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:29:55,523 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/utmzm97k-media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4
+2024-12-31 22:29:55,544 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:56,132 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:29:56,132 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:29:56,132 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4
+2024-12-31 22:29:56,132 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval
+2024-12-31 22:29:56,429 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/eu7mhhbv-media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4
+2024-12-31 22:29:56,546 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:57,282 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:29:57,545 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:58,551 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:29:59,551 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:00,133 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:00,546 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:01,569 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:02,548 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:02,570 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:03,546 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:04,134 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:04,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:05,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:06,549 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:07,549 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:08,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:30:08,524 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:30:08,569 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:08,592 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:09,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:10,136 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:30:10,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:10,996 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:30:10,996 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:30:10,996 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:30:11,136 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:30:11,136 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:30:11,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:12,137 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:12,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:13,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:13,904 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:14,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:15,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:16,138 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:16,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:17,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:18,552 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:19,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:19,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:20,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:21,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:22,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:23,194 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:30:23,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:30:23,520 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:30:23,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:24,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:24,627 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:25,553 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:26,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:27,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:28,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:29,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:30,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:30,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:31,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:32,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:33,554 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:34,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:35,526 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:35,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:36,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:37,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:38,519 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:30:38,520 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:30:38,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:39,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:40,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:40,596 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:41,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:42,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:43,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:44,555 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:45,556 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:46,528 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:46,556 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:47,558 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:48,557 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:49,560 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:50,062 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:30:50,062 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4 with policy now
+2024-12-31 22:30:50,063 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:30:50,063 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:30:50,063 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:30:50,065 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:30:50,149 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:30:50,149 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:50,149 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4
+2024-12-31 22:30:50,150 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval
+2024-12-31 22:30:50,360 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/y9cwp5qi-media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4
+2024-12-31 22:30:50,558 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:50,645 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/1cr9sa4t-media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4
+2024-12-31 22:30:51,561 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:52,101 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:52,150 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:52,568 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:53,195 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:30:53,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:30:53,522 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:30:53,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:54,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:55,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:56,151 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:56,565 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:57,579 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:57,617 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:30:58,152 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:30:58,566 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:59,152 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:30:59,428 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/gogcd3wz-events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:30:59,574 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:30:59,839 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:30:59,839 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:30:59,840 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:31:00,153 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:31:00,153 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:00,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:01,581 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:02,153 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:02,573 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:02,717 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:03,570 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:04,154 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:04,570 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:05,570 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:06,155 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:06,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:07,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:07,724 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:08,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:31:08,520 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:31:08,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:09,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:10,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:11,571 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:12,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:13,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:13,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:14,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:15,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:16,572 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:17,575 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:18,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:18,573 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:19,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:20,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:21,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:22,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:23,195 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:31:23,520 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:31:23,521 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:31:23,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:23,669 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:24,576 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:25,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:26,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:27,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:28,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:29,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:29,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:30,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:31,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:32,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:33,577 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:34,530 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:34,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:35,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:36,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:37,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:38,521 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:31:38,521 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:31:38,578 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:39,393 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:31:39,393 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4 with policy now
+2024-12-31 22:31:39,398 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:31:39,398 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:31:39,398 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:31:39,413 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:31:39,584 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:39,664 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/x7305ls9-media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4
+2024-12-31 22:31:40,166 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:31:40,166 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:40,166 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4
+2024-12-31 22:31:40,174 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval
+2024-12-31 22:31:40,450 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:40,496 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/2zh2ping-media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4
+2024-12-31 22:31:40,583 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:41,587 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:42,167 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:42,587 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:43,586 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:44,167 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:44,589 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:45,547 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:45,592 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:46,592 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:47,587 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:48,169 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:48,589 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:49,591 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:50,593 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:51,260 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:51,597 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:51,905 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:31:51,905 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:31:51,905 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:31:52,170 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:31:52,170 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:31:52,170 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:52,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:53,196 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:31:53,524 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:31:53,524 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:31:53,600 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:54,171 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:54,611 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:55,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:56,171 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:56,608 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:57,185 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:31:57,604 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:58,172 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:31:58,604 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:31:59,604 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:00,173 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:00,605 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:01,605 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:02,605 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:02,805 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:03,607 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:04,174 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:04,607 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:05,607 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:06,175 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:06,607 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:07,608 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:08,023 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:08,523 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:32:08,523 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:32:08,608 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:09,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:10,177 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:10,608 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:11,608 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:12,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:13,533 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:13,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:14,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:15,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:16,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:17,609 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:18,179 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:18,534 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:18,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:19,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:20,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:21,610 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:22,611 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:23,196 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:32:23,522 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:32:23,523 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:32:23,611 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:23,657 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:24,611 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:25,611 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:26,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:27,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:28,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:29,533 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:29,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:30,612 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:31,613 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:32,613 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:33,626 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:34,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:34,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
+2024-12-31 22:32:34,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval
+2024-12-31 22:32:34,205 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:32:34,205 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4 with policy now
+2024-12-31 22:32:34,210 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:32:34,211 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:32:34,211 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:34,211 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:34,494 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/90n6cfmi-media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
+2024-12-31 22:32:34,615 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:35,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
+2024-12-31 22:32:35,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:35,253 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:35,523 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/816v1cg4-media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
+2024-12-31 22:32:35,629 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:36,185 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:36,636 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:37,627 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:38,536 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 22:32:38,536 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 22:32:38,647 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:39,640 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:40,187 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:40,622 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:40,634 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:41,633 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:42,187 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:42,644 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:43,639 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:44,188 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:32:44,436 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/0ywzqbxo-events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:32:44,615 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:32:44,615 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:32:44,635 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:44,774 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:32:45,104 INFO    wandb-upload_0:1757972 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpco79vl2s
+2024-12-31 22:32:45,167 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 22:32:45,181 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table
+2024-12-31 22:32:45,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval
+2024-12-31 22:32:45,517 INFO    SenderThread:1757972 [sender.py:send_request_log_artifact():1525] logged artifact run-6rp1ws1k-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE2MzY4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MzI2MQ==', 'latestArtifact': None}}
+2024-12-31 22:32:45,517 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:32:45,517 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2024-12-31 22:32:45,517 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:32:45,517 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,518 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,518 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:32:45,518 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,522 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,522 DEBUG   SenderThread:1757972 [sender.py:send():379] send: telemetry
+2024-12-31 22:32:45,522 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,522 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,523 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,523 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,523 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,523 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,523 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,524 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,524 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,524 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,524 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,525 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,525 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,525 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,525 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,525 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,526 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,526 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,526 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,526 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,527 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:45,527 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:45,527 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 22:32:45,632 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:45,707 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/sl2foi94-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:32:45,718 INFO    wandb-upload_2:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/ahec4sm7-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json
+2024-12-31 22:32:45,884 INFO    wandb-upload_0:1757972 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplopob7gl
+2024-12-31 22:32:45,956 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/fwu04w86-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:32:46,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:46,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:46,286 INFO    SenderThread:1757972 [sender.py:send_request_log_artifact():1525] logged artifact run-6rp1ws1k-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDE2MzY5NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE3MzI2Mg==', 'latestArtifact': None}}
+2024-12-31 22:32:46,289 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:46,403 DEBUG   SenderThread:1757972 [sender.py:send():379] send: files
+2024-12-31 22:32:46,403 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json with policy now
+2024-12-31 22:32:46,403 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:32:46,403 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:46,404 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:46,633 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:46,670 INFO    wandb-upload_2:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/970y0upf-media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json
+2024-12-31 22:32:47,189 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:47,190 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml
+2024-12-31 22:32:47,632 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:48,190 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:48,633 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:49,633 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:49,909 DEBUG   SenderThread:1757972 [sender.py:send():379] send: exit
+2024-12-31 22:32:49,909 INFO    SenderThread:1757972 [sender.py:send_exit():586] handling exit code: 0
+2024-12-31 22:32:49,910 INFO    SenderThread:1757972 [sender.py:send_exit():588] handling runtime: 296
+2024-12-31 22:32:49,910 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:49,910 INFO    SenderThread:1757972 [sender.py:send_exit():594] send defer
+2024-12-31 22:32:49,910 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:49,910 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-31 22:32:49,910 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:49,910 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-31 22:32:49,910 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 1
+2024-12-31 22:32:49,911 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:49,911 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-31 22:32:49,911 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:49,911 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-31 22:32:49,911 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 2
+2024-12-31 22:32:49,911 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:49,911 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-31 22:32:49,911 INFO    HandlerThread:1757972 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-31 22:32:49,911 DEBUG   SystemMonitor:1757972 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-31 22:32:49,911 DEBUG   SystemMonitor:1757972 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-31 22:32:49,911 INFO    HandlerThread:1757972 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-31 22:32:49,912 INFO    HandlerThread:1757972 [interfaces.py:finish():200] Joined disk monitor
+2024-12-31 22:32:49,981 INFO    HandlerThread:1757972 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-31 22:32:49,981 INFO    HandlerThread:1757972 [interfaces.py:finish():200] Joined memory monitor
+2024-12-31 22:32:49,982 INFO    HandlerThread:1757972 [interfaces.py:finish():200] Joined network monitor
+2024-12-31 22:32:49,982 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:49,982 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-31 22:32:49,982 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 3
+2024-12-31 22:32:49,982 DEBUG   SenderThread:1757972 [sender.py:send():379] send: stats
+2024-12-31 22:32:49,982 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:49,982 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-31 22:32:49,983 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:49,983 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-31 22:32:49,983 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 4
+2024-12-31 22:32:49,983 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:49,983 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-31 22:32:50,191 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:52,192 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:56,978 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:32:56,978 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:56,978 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-31 22:32:56,979 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:56,979 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-31 22:32:56,979 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 5
+2024-12-31 22:32:56,979 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:32:56,979 DEBUG   SenderThread:1757972 [sender.py:send():379] send: history
+2024-12-31 22:32:56,979 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 22:32:56,979 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:56,979 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-31 22:32:56,980 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:56,980 DEBUG   SenderThread:1757972 [sender.py:send():379] send: summary
+2024-12-31 22:32:56,984 INFO    SenderThread:1757972 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 22:32:56,984 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:56,984 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-31 22:32:56,984 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:56,984 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 6
+2024-12-31 22:32:56,984 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:56,984 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-31 22:32:56,985 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:56,985 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-31 22:32:56,985 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 7
+2024-12-31 22:32:56,985 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 22:32:56,985 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:56,985 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-31 22:32:56,985 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:56,985 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-31 22:32:57,194 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:57,911 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:32:57,989 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 8
+2024-12-31 22:32:57,989 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:32:57,989 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:57,989 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-31 22:32:57,990 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:57,990 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-31 22:32:57,990 INFO    SenderThread:1757972 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-31 22:32:57,992 INFO    SenderThread:1757972 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-31 22:32:57,994 INFO    SenderThread:1757972 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-31 22:32:58,011 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 9
+2024-12-31 22:32:58,011 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:58,011 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-31 22:32:58,012 DEBUG   SenderThread:1757972 [sender.py:send():379] send: artifact
+2024-12-31 22:32:58,195 INFO    Thread-12 :1757972 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:58,410 INFO    wandb-upload_2:1757972 [upload_job.py:push():85] Skipped uploading /tmp/tmpqrps9wt9/wandb-job.json
+2024-12-31 22:32:58,410 INFO    wandb-upload_0:1757972 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmmq90y7h
+2024-12-31 22:32:58,530 INFO    wandb-upload_1:1757972 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2s46j5eq
+2024-12-31 22:32:58,912 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:32:59,161 INFO    SenderThread:1757972 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDE2MzgxOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDE1OTY3NQ==', 'versionIndex': 396}}}
+2024-12-31 22:32:59,161 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,161 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-31 22:32:59,161 INFO    SenderThread:1757972 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-31 22:32:59,196 INFO    SenderThread:1757972 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files
+2024-12-31 22:32:59,196 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml config.yaml
+2024-12-31 22:32:59,196 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json wandb-summary.json
+2024-12-31 22:32:59,197 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0 events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/requirements.txt requirements.txt
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-metadata.json wandb-metadata.json
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log output.log
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/diff.patch diff.patch
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4 media/videos/eval/0_eval_0_037a1e72b853dceefbcd.mp4
+2024-12-31 22:32:59,200 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4 media/videos/eval/3_eval_3_0ed43f7879c76b869ca4.mp4
+2024-12-31 22:32:59,201 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4 media/videos/eval/4_eval_4_e9d47729d90bd1ce9981.mp4
+2024-12-31 22:32:59,201 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4 media/videos/eval/1_eval_1_2cd7d5b3a1a16852a937.mp4
+2024-12-31 22:32:59,201 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4 media/videos/eval/2_eval_2_6c119cb8d8d9251fce33.mp4
+2024-12-31 22:32:59,204 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json media/table/eval/total_success_rate_ci_6_633be57ab8f5d2a03a6e.table.json
+2024-12-31 22:32:59,204 INFO    SenderThread:1757972 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2024-12-31 22:32:59,204 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 10
+2024-12-31 22:32:59,204 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:32:59,204 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:59,204 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-31 22:32:59,206 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,206 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-31 22:32:59,206 INFO    SenderThread:1757972 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:32:59,379 INFO    wandb-upload_0:1757972 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/wandb-summary.json
+2024-12-31 22:32:59,399 INFO    wandb-upload_2:1757972 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/config.yaml
+2024-12-31 22:32:59,432 INFO    wandb-upload_4:1757972 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/output.log
+2024-12-31 22:32:59,449 INFO    wandb-upload_1:1757972 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/files/requirements.txt
+2024-12-31 22:32:59,496 INFO    wandb-upload_3:1757972 [upload_job.py:push():130] Uploaded file /tmp/tmpi3e0i062wandb/uo92ha99-events.out.tfevents.1735702073.leonmkim-ROG-Strix-G15CS-G15CS.1757775.0
+2024-12-31 22:32:59,696 INFO    Thread-11 (_thread_body):1757972 [sender.py:transition_state():614] send defer: 11
+2024-12-31 22:32:59,696 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:59,696 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-31 22:32:59,697 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,697 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-31 22:32:59,697 INFO    SenderThread:1757972 [file_pusher.py:join():175] waiting for file pusher
+2024-12-31 22:32:59,698 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 12
+2024-12-31 22:32:59,698 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:59,698 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-31 22:32:59,698 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,698 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-31 22:32:59,698 INFO    SenderThread:1757972 [file_stream.py:finish():601] file stream finish called
+2024-12-31 22:32:59,895 INFO    SenderThread:1757972 [file_stream.py:finish():605] file stream finish is done
+2024-12-31 22:32:59,895 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 13
+2024-12-31 22:32:59,895 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:59,895 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-31 22:32:59,895 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,895 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-31 22:32:59,895 INFO    SenderThread:1757972 [sender.py:transition_state():614] send defer: 14
+2024-12-31 22:32:59,896 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 22:32:59,896 INFO    HandlerThread:1757972 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-31 22:32:59,896 DEBUG   SenderThread:1757972 [sender.py:send():379] send: final
+2024-12-31 22:32:59,896 DEBUG   SenderThread:1757972 [sender.py:send():379] send: footer
+2024-12-31 22:32:59,896 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: defer
+2024-12-31 22:32:59,896 INFO    SenderThread:1757972 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-31 22:32:59,896 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:32:59,897 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:32:59,897 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: server_info
+2024-12-31 22:32:59,897 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 22:32:59,897 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-31 22:32:59,898 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-31 22:32:59,899 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 22:32:59,899 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: server_info
+2024-12-31 22:32:59,949 DEBUG   SenderThread:1757972 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 22:32:59,950 INFO    MainThread:1757972 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-31 22:32:59,950 INFO    MainThread:1757972 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-31 22:32:59,950 INFO    MainThread:1757972 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-31 22:32:59,951 DEBUG   HandlerThread:1757972 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-31 22:32:59,951 INFO    HandlerThread:1757972 [handler.py:finish():882] shutting down handler
+2024-12-31 22:33:00,899 INFO    WriterThread:1757972 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/run-6rp1ws1k.wandb
+2024-12-31 22:33:00,950 INFO    SenderThread:1757972 [sender.py:finish():1615] shutting down sender
+2024-12-31 22:33:00,950 INFO    SenderThread:1757972 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 22:33:00,950 INFO    SenderThread:1757972 [file_pusher.py:join():175] waiting for file pusher
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/logs/debug.log b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..b9c1f72b10620f06ceb0d5087e483ba6d43c4fde
--- /dev/null
+++ b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/logs/debug.log
@@ -0,0 +1,31 @@
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Configure stats pid to 1757775
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/settings
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-31 22:27:52,837 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/logs/debug.log
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/logs/debug-internal.log
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:init():569] calling init triggers
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k'}
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:init():619] starting backend
+2024-12-31 22:27:52,838 INFO    MainThread:1757775 [wandb_init.py:init():623] setting up manager
+2024-12-31 22:27:52,842 INFO    MainThread:1757775 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-31 22:27:52,843 INFO    MainThread:1757775 [wandb_init.py:init():631] backend started and connected
+2024-12-31 22:27:52,856 INFO    MainThread:1757775 [wandb_init.py:init():720] updated telemetry
+2024-12-31 22:27:52,862 INFO    MainThread:1757775 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-31 22:27:53,113 INFO    MainThread:1757775 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-31 22:27:53,179 INFO    MainThread:1757775 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-31 22:27:53,179 INFO    MainThread:1757775 [wandb_init.py:init():804] starting run threads in backend
+2024-12-31 22:27:53,518 INFO    MainThread:1757775 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-31 22:27:53,518 INFO    MainThread:1757775 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-31 22:27:53,518 INFO    MainThread:1757775 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-31 22:27:53,518 INFO    MainThread:1757775 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-31 22:27:53,520 INFO    MainThread:1757775 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-31 22:27:53,520 INFO    MainThread:1757775 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/6rp1ws1k/tb, True
+2024-12-31 22:27:58,590 INFO    MainThread:1757775 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-31 22:33:00,951 WARNING MsgRouterThr:1757775 [router.py:message_loop():77] message_loop has been closed
diff --git a/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/run-6rp1ws1k.wandb b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/run-6rp1ws1k.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..a34028da5676bb2824cfa1e7b7e7b06aee52720c
Binary files /dev/null and b/6rp1ws1k/wandb/run-20241231_222752-6rp1ws1k/run-6rp1ws1k.wandb differ
diff --git a/7j9cp4xk/episode_rosbags/episode_2_2025-01-01-00-10-52.bag b/7j9cp4xk/episode_rosbags/episode_2_2025-01-01-00-10-52.bag
new file mode 100644
index 0000000000000000000000000000000000000000..5e316d8449e5b0a70dad5a0d0f5afcd53e4676b5
--- /dev/null
+++ b/7j9cp4xk/episode_rosbags/episode_2_2025-01-01-00-10-52.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fc1e9f318e2395faebd4055dba0334578a97e24bdf51ccc91df57feb452dce4
+size 1429826011
diff --git a/7j9cp4xk/episode_rosbags/episode_3_2025-01-01-00-11-48.bag b/7j9cp4xk/episode_rosbags/episode_3_2025-01-01-00-11-48.bag
new file mode 100644
index 0000000000000000000000000000000000000000..71b5ed605e02202bae7738f18c51bf19feee4774
--- /dev/null
+++ b/7j9cp4xk/episode_rosbags/episode_3_2025-01-01-00-11-48.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:865a35b782208f42686e3d875b7f2da71d626cf11f3363d9a47c21a3e6d37b1e
+size 1421412982
diff --git a/c7zbixne/.hydra/config.yaml b/c7zbixne/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90
--- /dev/null
+++ b/c7zbixne/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1006_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/c7zbixne/.hydra/hydra.yaml b/c7zbixne/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d48e7bbf1e57d802745e354bf119256387d43d23
--- /dev/null
+++ b/c7zbixne/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/c7zbixne/.hydra/overrides.yaml b/c7zbixne/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/c7zbixne/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/c7zbixne/episode_rosbags/aligned_depth_to_color_K.npy b/c7zbixne/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/c7zbixne/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/c7zbixne/episode_rosbags/cam_tf_world.npy b/c7zbixne/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/c7zbixne/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/c7zbixne/episode_rosbags/color_K.npy b/c7zbixne/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/c7zbixne/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/c7zbixne/episode_rosbags/depth_K.npy b/c7zbixne/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/c7zbixne/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/c7zbixne/episode_rosbags/episode_1_2024-12-31-21-36-35.bag b/c7zbixne/episode_rosbags/episode_1_2024-12-31-21-36-35.bag
new file mode 100644
index 0000000000000000000000000000000000000000..1f50bce72c11974c6f0f46bc3fa61ff4a57fca38
--- /dev/null
+++ b/c7zbixne/episode_rosbags/episode_1_2024-12-31-21-36-35.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d83cd79c1a4f8e7de79edce214d3650078278c8867aff48e9fc7958c9579ecb
+size 938054375
diff --git a/c7zbixne/episode_rosbags/episode_2_2024-12-31-21-37-20.bag b/c7zbixne/episode_rosbags/episode_2_2024-12-31-21-37-20.bag
new file mode 100644
index 0000000000000000000000000000000000000000..2d70675bb0e04467d7c4ec786ebc583ffe97a148
--- /dev/null
+++ b/c7zbixne/episode_rosbags/episode_2_2024-12-31-21-37-20.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fa545afabbb4c646076ebb5519be0423412d401303228b0e0a30bb2d98984fa
+size 1748666355
diff --git a/c7zbixne/eval_robot.log b/c7zbixne/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..72ba14021c53cf730ba5d48725e05335526931fd
--- /dev/null
+++ b/c7zbixne/eval_robot.log
@@ -0,0 +1,12 @@
+[2024-12-31 21:34:57,574][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-31 21:34:57,577][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/c7zbixne/eval_video/0_eval.mp4 b/c7zbixne/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1606c79932554d899f77ab0de41f643ec5bad6b5
--- /dev/null
+++ b/c7zbixne/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8a408e3df7bfc74cc89f4bb1e1a72dba36827787ddaf45bcdc22e92b20b160b
+size 1465413
diff --git a/c7zbixne/eval_video/1_eval.mp4 b/c7zbixne/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9ba595fe3a153cf73d7f9f43206af10791b34879
--- /dev/null
+++ b/c7zbixne/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb902dfcfac85de572711b69b5169f14cd46b193216b8d0fb49f7108c7df956a
+size 609556
diff --git a/c7zbixne/eval_video/2_eval.mp4 b/c7zbixne/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..6423327b771a47e803c87a90aca4f8b8292d2927
--- /dev/null
+++ b/c7zbixne/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dea02de20bdd586cd5fdc45e12e91aa0201c0f251b47214c0c207f72e26d4bbf
+size 1059657
diff --git a/c7zbixne/eval_video/3_eval.mp4 b/c7zbixne/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..70dda4306f1be91d1e747f4eadd35d0847dc4321
--- /dev/null
+++ b/c7zbixne/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d10ed98fe5d597fa5a89e7f68b60f25182c82904b1d424cc5d6328ea83717bda
+size 1466419
diff --git a/c7zbixne/eval_video/4_eval.mp4 b/c7zbixne/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bb3721983cf3b167e0829a2368557500e6f8c76f
--- /dev/null
+++ b/c7zbixne/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:578f39c6eeca4936625ce4639b6f651b2683d682a9c827e509298a184631d901
+size 1516972
diff --git a/c7zbixne/tb/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0 b/c7zbixne/tb/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
new file mode 100644
index 0000000000000000000000000000000000000000..f5cb743303bfa53b6b61f73496f709dccb76252f
--- /dev/null
+++ b/c7zbixne/tb/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99d8a9f9e1a8e0f3d887a8253f6308e2395022d5f4750098d01599107cd9b9bd
+size 6539
diff --git a/c7zbixne/wandb/debug-internal.log b/c7zbixne/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..3d270f1304aa0a4788b0653ac6b57efc21702435
--- /dev/null
+++ b/c7zbixne/wandb/debug-internal.log
@@ -0,0 +1,883 @@
+2024-12-31 21:35:03,442 INFO    StreamThr :1748203 [internal.py:wandb_internal():85] W&B internal server running at pid: 1748203, started at: 2024-12-31 21:35:03.441695
+2024-12-31 21:35:03,443 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status
+2024-12-31 21:35:03,453 INFO    WriterThread:1748203 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/run-c7zbixne.wandb
+2024-12-31 21:35:03,454 DEBUG   SenderThread:1748203 [sender.py:send():379] send: header
+2024-12-31 21:35:03,464 DEBUG   SenderThread:1748203 [sender.py:send():379] send: run
+2024-12-31 21:35:03,684 INFO    SenderThread:1748203 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files
+2024-12-31 21:35:03,684 INFO    SenderThread:1748203 [sender.py:_start_run_threads():1188] run started: c7zbixne with start time 1735698903.442219
+2024-12-31 21:35:03,701 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: check_version
+2024-12-31 21:35:03,701 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: check_version
+2024-12-31 21:35:03,776 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: run_start
+2024-12-31 21:35:03,785 DEBUG   HandlerThread:1748203 [system_info.py:__init__():26] System info init
+2024-12-31 21:35:03,785 DEBUG   HandlerThread:1748203 [system_info.py:__init__():41] System info init done
+2024-12-31 21:35:03,785 INFO    HandlerThread:1748203 [system_monitor.py:start():194] Starting system monitor
+2024-12-31 21:35:03,785 INFO    SystemMonitor:1748203 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-31 21:35:03,785 INFO    HandlerThread:1748203 [system_monitor.py:probe():214] Collecting system info
+2024-12-31 21:35:03,785 INFO    SystemMonitor:1748203 [interfaces.py:start():188] Started cpu monitoring
+2024-12-31 21:35:03,786 INFO    SystemMonitor:1748203 [interfaces.py:start():188] Started disk monitoring
+2024-12-31 21:35:03,787 INFO    SystemMonitor:1748203 [interfaces.py:start():188] Started gpu monitoring
+2024-12-31 21:35:03,788 INFO    SystemMonitor:1748203 [interfaces.py:start():188] Started memory monitoring
+2024-12-31 21:35:03,788 INFO    SystemMonitor:1748203 [interfaces.py:start():188] Started network monitoring
+2024-12-31 21:35:03,799 DEBUG   HandlerThread:1748203 [system_info.py:probe():152] Probing system
+2024-12-31 21:35:03,800 DEBUG   HandlerThread:1748203 [system_info.py:_probe_git():137] Probing git
+2024-12-31 21:35:03,805 DEBUG   HandlerThread:1748203 [system_info.py:_probe_git():145] Probing git done
+2024-12-31 21:35:03,805 DEBUG   HandlerThread:1748203 [system_info.py:probe():200] Probing system done
+2024-12-31 21:35:03,805 DEBUG   HandlerThread:1748203 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:35:03.799114', 'startedAt': '2025-01-01T02:35:03.434313', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.55275, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.978, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 515.321044921875}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-31 21:35:03,805 INFO    HandlerThread:1748203 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-31 21:35:03,805 INFO    HandlerThread:1748203 [system_monitor.py:probe():227] Publishing system info
+2024-12-31 21:35:03,805 DEBUG   HandlerThread:1748203 [system_info.py:_save_code():44] Saving code
+2024-12-31 21:35:03,806 DEBUG   HandlerThread:1748203 [system_info.py:_save_code():67] Saving code done
+2024-12-31 21:35:03,806 DEBUG   HandlerThread:1748203 [system_info.py:_save_patches():84] Saving git patches
+2024-12-31 21:35:03,868 DEBUG   HandlerThread:1748203 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-31 21:35:03,869 INFO    HandlerThread:1748203 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-31 21:35:03,872 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:35:03,872 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-31 21:35:03,872 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-31 21:35:03,872 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-31 21:35:04,121 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-31 21:35:04,121 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: python_packages
+2024-12-31 21:35:04,121 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:35:04,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:35:04,124 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/fcqxte4d-diff.patch
+2024-12-31 21:35:04,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:04,125 INFO    HandlerThread:1748203 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/tb"
+  save: true
+}
+control {
+  relay_id: "0766f48012f149d2be96e80bee70b554"
+}
+_info {
+  stream_id: "c7zbixne"
+}
+
+2024-12-31 21:35:04,126 DEBUG   HandlerThread:1748203 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-31 21:35:04,159 INFO    wandb-upload_0:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/xj2oosdr-wandb-metadata.json
+2024-12-31 21:35:04,187 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/qe78rsai-code/FISH/eval_robot.py
+2024-12-31 21:35:04,284 DEBUG   SenderThread:1748203 [sender.py:send():379] send: telemetry
+2024-12-31 21:35:04,284 DEBUG   SenderThread:1748203 [sender.py:send():379] send: tbrecord
+2024-12-31 21:35:04,285 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:35:04,285 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0 with policy live
+2024-12-31 21:35:04,519 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/jafedgkg-events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:35:04,685 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-metadata.json
+2024-12-31 21:35:04,685 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/diff.patch
+2024-12-31 21:35:04,685 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code/FISH/eval_robot.py
+2024-12-31 21:35:04,686 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:35:04,686 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/requirements.txt
+2024-12-31 21:35:04,686 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:04,686 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code/FISH
+2024-12-31 21:35:04,686 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code
+2024-12-31 21:35:05,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:06,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:06,684 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:07,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:08,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:09,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:09,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:10,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:11,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:12,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:13,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:14,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:14,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:15,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:16,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:17,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:18,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:18,759 DEBUG   SenderThread:1748203 [sender.py:send():379] send: config
+2024-12-31 21:35:19,121 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:35:19,121 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:35:19,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:19,274 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:20,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:20,689 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:21,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:22,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:22,690 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:23,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:24,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:24,690 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:25,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:25,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:26,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:27,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:28,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:28,691 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:29,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:30,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:30,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:31,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:32,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:32,692 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:33,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:34,121 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:35:34,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:35:34,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:35,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:35,331 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:35,693 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml
+2024-12-31 21:35:36,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:36,694 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:37,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:38,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:38,694 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:39,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:40,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:40,451 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:40,695 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:41,127 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:42,127 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:42,695 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:43,127 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:44,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:44,696 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:45,127 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:45,520 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:46,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:46,697 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:47,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:48,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:48,697 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:49,121 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:35:49,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:35:49,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:50,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:50,639 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:50,698 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:51,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:52,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:52,698 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:53,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:54,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:54,699 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:55,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:55,841 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:56,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:56,700 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:57,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:58,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:58,700 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:59,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:00,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:00,701 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:01,117 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:01,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:02,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:02,701 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:03,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:03,788 DEBUG   SystemMonitor:1748203 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-31 21:36:03,789 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:36:04,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:36:04,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:36:04,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:04,702 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:05,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:06,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:06,354 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:06,702 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:07,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:08,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:08,703 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:09,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:10,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:10,372 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:36:10,372 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4 with policy now
+2024-12-31 21:36:10,372 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:36:10,373 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:36:10,373 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:36:10,374 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:36:10,588 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/ii2pmaby-media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos
+2024-12-31 21:36:10,961 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/t9ibgwop-media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4
+2024-12-31 21:36:11,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:11,389 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:12,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:12,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:13,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:14,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:15,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:16,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:16,705 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:16,706 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:17,132 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:18,132 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:18,706 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:19,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:36:19,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:36:19,165 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:20,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:21,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:22,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:22,206 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:23,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:24,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:25,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:26,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:27,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:28,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:28,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:29,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:30,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:31,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:32,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:32,711 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:36:32,947 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/6vmh7e8o-events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:36:33,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:33,789 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:36:33,790 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:34,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:36:34,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:36:34,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:34,712 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:35,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:36,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:36,712 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:37,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:38,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:38,713 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:38,987 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:39,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:40,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:40,714 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:41,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:42,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:42,715 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:43,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:44,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:44,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:44,716 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:45,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:46,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:46,716 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:47,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:48,137 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:48,717 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:49,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:36:49,126 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:36:49,137 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:49,248 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:49,970 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:36:49,970 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4 with policy now
+2024-12-31 21:36:49,972 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:36:49,973 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:36:49,973 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:36:49,973 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:36:50,137 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:50,222 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/8z93u7al-media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4
+2024-12-31 21:36:50,718 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:50,718 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:36:50,718 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4
+2024-12-31 21:36:50,718 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval
+2024-12-31 21:36:50,978 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/8lbc22xp-media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4
+2024-12-31 21:36:51,141 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:52,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:52,718 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:53,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:54,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:54,719 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:55,127 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:55,145 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:56,138 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:57,139 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:58,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:58,720 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:59,145 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:00,145 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:01,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:01,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:02,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:03,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:03,790 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:37:04,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:37:04,123 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:37:04,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:05,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:06,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:06,236 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:07,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:08,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:09,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:10,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:11,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:12,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:12,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:13,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:14,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:15,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:16,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:17,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:17,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:17,578 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:37:17,578 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:37:17,579 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:37:17,727 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:37:17,727 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:37:18,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:18,727 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:19,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:37:19,123 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:37:19,165 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:20,145 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:20,728 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:21,145 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:22,150 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:22,302 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:22,728 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:23,149 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:24,150 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:24,729 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:25,151 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:26,151 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:26,730 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:27,151 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:27,478 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:28,151 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:28,731 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:29,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:30,151 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:30,731 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:31,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:32,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:32,655 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:32,732 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:33,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:33,790 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:37:34,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:37:34,125 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:37:34,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:34,733 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:35,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:36,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:36,734 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:37,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:37,856 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:38,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:38,734 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:39,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:40,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:40,735 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:41,153 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:42,154 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:42,736 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:43,078 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:43,153 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:44,177 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:44,737 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:45,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:45,908 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:37:45,908 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4 with policy now
+2024-12-31 21:37:45,913 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:37:45,913 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:37:45,913 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:37:45,914 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:37:46,119 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/ju9rtktg-media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4
+2024-12-31 21:37:46,153 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:46,737 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:46,738 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:37:46,738 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4
+2024-12-31 21:37:46,738 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval
+2024-12-31 21:37:47,001 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/fvizvw0x-media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4
+2024-12-31 21:37:47,158 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:48,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:48,738 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:48,945 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:49,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:37:49,125 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:37:49,165 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:50,157 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:50,739 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:51,157 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:52,156 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:52,739 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:53,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:54,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:54,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:55,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:56,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:57,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:58,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:59,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:59,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:00,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:01,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:02,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:03,163 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:03,791 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:38:04,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:38:04,125 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:38:04,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:04,290 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:04,414 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:38:04,414 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:38:04,414 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:38:04,743 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:38:04,744 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:04,744 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:38:04,991 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/abujme1x-events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:38:05,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:06,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:06,744 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:07,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:08,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:08,745 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:09,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:09,336 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:10,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:10,745 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:11,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:12,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:12,746 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:13,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:14,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:14,585 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:14,747 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:15,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:16,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:16,748 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:17,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:18,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:18,748 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:19,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:38:19,125 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:38:19,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:19,759 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:20,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:20,749 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:21,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:22,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:22,750 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:23,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:24,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:24,752 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:25,004 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:25,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:26,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:26,752 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:27,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:28,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:28,753 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:29,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:30,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:30,210 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:30,754 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:31,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:32,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:32,755 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:33,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:33,791 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:38:34,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:38:34,125 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:38:34,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:34,756 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:35,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:35,460 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:36,170 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:36,756 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:37,170 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:38,170 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:38,757 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:39,170 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:40,170 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:40,684 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:40,758 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:41,171 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:42,171 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:42,758 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:43,175 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:43,962 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:38:43,962 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4 with policy now
+2024-12-31 21:38:43,970 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:38:43,971 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:38:43,971 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:38:43,978 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:38:44,172 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:44,184 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/2up8vcgt-media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4
+2024-12-31 21:38:44,759 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:44,759 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:38:44,759 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4
+2024-12-31 21:38:44,760 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval
+2024-12-31 21:38:45,051 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/f95qu0cn-media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4
+2024-12-31 21:38:45,177 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:46,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:46,182 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:47,187 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:48,179 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:48,761 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:49,132 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:38:49,132 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:38:49,188 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:50,183 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:51,183 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:51,465 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:52,193 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:52,762 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:53,185 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:54,194 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:55,192 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:56,193 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:57,139 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:57,187 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:58,195 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:59,199 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:00,195 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:01,200 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:02,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:03,138 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:03,196 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:03,792 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:39:04,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:39:04,133 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:39:04,195 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:05,201 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:06,197 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:06,487 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:39:06,487 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:39:06,487 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:39:06,767 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:39:06,767 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:06,767 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:39:07,019 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/4bycnqb2-events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:39:07,202 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:08,165 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:08,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:08,767 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:09,202 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:10,202 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:10,768 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:11,202 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:12,203 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:12,769 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:13,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:13,337 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:14,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:14,770 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:15,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:16,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:16,770 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:17,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:18,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:18,526 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:18,771 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:19,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:39:19,128 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:39:19,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:20,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:20,772 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:21,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:22,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:22,773 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:23,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:23,528 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:24,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:24,773 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:25,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:26,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:26,774 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:27,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:28,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:28,722 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:28,775 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:29,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:30,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:30,776 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:31,206 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:32,206 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:32,776 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:33,207 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:33,792 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:39:33,792 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:34,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:39:34,128 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:39:34,207 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:34,777 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:35,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:36,207 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:36,778 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:37,207 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:38,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:38,779 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:38,897 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:39,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:40,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:40,779 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:41,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:42,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:42,780 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:43,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:44,094 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:44,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:44,781 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:45,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:46,217 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:46,326 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:39:46,326 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4 with policy now
+2024-12-31 21:39:46,345 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:39:46,346 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:39:46,346 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:39:46,364 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:39:46,546 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/gxxmyc6m-media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4
+2024-12-31 21:39:46,782 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:46,782 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:39:46,782 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4
+2024-12-31 21:39:46,782 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval
+2024-12-31 21:39:47,046 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/wr4ci5gj-media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4
+2024-12-31 21:39:47,239 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:48,246 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:48,782 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:49,132 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:39:49,132 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:39:49,234 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:49,287 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:50,237 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:50,783 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:51,241 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:52,241 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:53,241 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:54,243 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:54,785 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:55,148 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:55,270 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:56,253 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:57,250 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:58,252 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:59,257 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:00,251 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:01,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:40:01,260 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:02,252 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:03,262 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:03,792 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:40:04,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:40:04,132 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:40:04,259 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:04,788 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:40:05,165 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 21:40:05,166 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 21:40:05,262 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:05,431 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:40:05,626 INFO    wandb-upload_2:1748203 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8715f46y
+2024-12-31 21:40:05,784 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 21:40:05,788 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:40:05,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json
+2024-12-31 21:40:05,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json
+2024-12-31 21:40:05,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media
+2024-12-31 21:40:05,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval
+2024-12-31 21:40:05,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table
+2024-12-31 21:40:05,795 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:40:06,026 INFO    SenderThread:1748203 [sender.py:send_request_log_artifact():1525] logged artifact run-c7zbixne-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMzI0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2OTAxMg==', 'latestArtifact': None}}
+2024-12-31 21:40:06,026 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:40:06,026 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now
+2024-12-31 21:40:06,027 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:40:06,028 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,029 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,029 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:40:06,029 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,030 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,030 DEBUG   SenderThread:1748203 [sender.py:send():379] send: telemetry
+2024-12-31 21:40:06,031 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,031 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,032 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,032 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,033 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,033 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,034 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,034 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,034 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,035 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,035 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,035 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,035 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,036 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,036 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,036 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,036 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,037 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,037 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,037 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,037 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,038 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,038 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 21:40:06,219 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/m7nfgzio-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json
+2024-12-31 21:40:06,243 INFO    wandb-upload_0:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/mu41qi4m-media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json
+2024-12-31 21:40:06,255 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:06,459 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/hh7yudnl-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json
+2024-12-31 21:40:06,506 INFO    wandb-upload_1:1748203 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpv5ousy4x
+2024-12-31 21:40:06,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:40:06,874 INFO    SenderThread:1748203 [sender.py:send_request_log_artifact():1525] logged artifact run-c7zbixne-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMzI2Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2OTAxNw==', 'latestArtifact': None}}
+2024-12-31 21:40:06,874 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:40:06,874 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json with policy now
+2024-12-31 21:40:06,875 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:40:06,875 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,875 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:40:06,876 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:07,117 INFO    wandb-upload_0:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/cy3wmssk-media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json
+2024-12-31 21:40:07,255 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:07,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:40:07,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:40:08,255 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:09,255 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:09,501 DEBUG   SenderThread:1748203 [sender.py:send():379] send: exit
+2024-12-31 21:40:09,501 INFO    SenderThread:1748203 [sender.py:send_exit():586] handling exit code: 0
+2024-12-31 21:40:09,502 INFO    SenderThread:1748203 [sender.py:send_exit():588] handling runtime: 305
+2024-12-31 21:40:09,502 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:09,502 INFO    SenderThread:1748203 [sender.py:send_exit():594] send defer
+2024-12-31 21:40:09,502 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:09,502 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-31 21:40:09,502 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:09,502 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-31 21:40:09,502 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 1
+2024-12-31 21:40:09,502 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:09,503 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-31 21:40:09,503 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:09,503 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-31 21:40:09,503 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 2
+2024-12-31 21:40:09,503 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:09,503 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-31 21:40:09,503 INFO    HandlerThread:1748203 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-31 21:40:09,503 DEBUG   SystemMonitor:1748203 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-31 21:40:09,503 DEBUG   SystemMonitor:1748203 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-31 21:40:09,504 INFO    HandlerThread:1748203 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-31 21:40:09,504 INFO    HandlerThread:1748203 [interfaces.py:finish():200] Joined disk monitor
+2024-12-31 21:40:09,556 INFO    HandlerThread:1748203 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-31 21:40:09,556 INFO    HandlerThread:1748203 [interfaces.py:finish():200] Joined memory monitor
+2024-12-31 21:40:09,556 INFO    HandlerThread:1748203 [interfaces.py:finish():200] Joined network monitor
+2024-12-31 21:40:09,557 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:09,557 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-31 21:40:09,557 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 3
+2024-12-31 21:40:09,557 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:40:09,557 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:09,557 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-31 21:40:09,557 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:09,557 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-31 21:40:09,557 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 4
+2024-12-31 21:40:09,557 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:09,557 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-31 21:40:09,790 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:40:11,791 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:40:15,561 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:15,561 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:40:15,561 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-31 21:40:15,562 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:15,562 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-31 21:40:15,562 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 5
+2024-12-31 21:40:15,563 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:15,563 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:40:15,563 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:15,563 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:15,563 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-31 21:40:15,564 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:15,564 DEBUG   SenderThread:1748203 [sender.py:send():379] send: summary
+2024-12-31 21:40:15,568 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:15,568 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:15,568 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-31 21:40:15,568 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 6
+2024-12-31 21:40:15,568 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:15,568 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-31 21:40:15,568 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:15,568 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-31 21:40:15,572 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:40:15,824 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 7
+2024-12-31 21:40:15,824 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:15,825 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-31 21:40:15,825 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:15,825 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-31 21:40:15,825 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:40:16,502 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:16,825 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml
+2024-12-31 21:40:18,018 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 8
+2024-12-31 21:40:18,019 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:18,019 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:18,019 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-31 21:40:18,019 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:18,019 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-31 21:40:18,019 INFO    SenderThread:1748203 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-31 21:40:18,021 INFO    SenderThread:1748203 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-31 21:40:18,023 INFO    SenderThread:1748203 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-31 21:40:18,031 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 9
+2024-12-31 21:40:18,031 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:18,032 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-31 21:40:18,032 DEBUG   SenderThread:1748203 [sender.py:send():379] send: artifact
+2024-12-31 21:40:18,448 INFO    SenderThread:1748203 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDIwNA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDIwNA==', 'versionIndex': 393}}}
+2024-12-31 21:40:18,448 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:18,448 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-31 21:40:18,448 INFO    SenderThread:1748203 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-31 21:40:18,503 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml config.yaml
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0 events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json wandb-summary.json
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/requirements.txt requirements.txt
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-metadata.json wandb-metadata.json
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log output.log
+2024-12-31 21:40:18,828 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/diff.patch diff.patch
+2024-12-31 21:40:18,828 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-31 21:40:18,829 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4 media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4
+2024-12-31 21:40:18,830 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4 media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4
+2024-12-31 21:40:18,830 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4 media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4
+2024-12-31 21:40:18,830 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4 media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4
+2024-12-31 21:40:18,830 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4 media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4
+2024-12-31 21:40:18,834 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json
+2024-12-31 21:40:18,835 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json
+2024-12-31 21:40:18,835 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 10
+2024-12-31 21:40:18,835 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:18,835 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:18,835 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-31 21:40:18,837 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:18,837 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-31 21:40:18,837 INFO    SenderThread:1748203 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 21:40:19,007 INFO    wandb-upload_0:1748203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:40:19,008 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml
+2024-12-31 21:40:19,064 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/requirements.txt
+2024-12-31 21:40:19,074 INFO    wandb-upload_4:1748203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:40:19,110 INFO    wandb-upload_3:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/edl5p080-events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:40:19,310 INFO    Thread-11 (_thread_body):1748203 [sender.py:transition_state():614] send defer: 11
+2024-12-31 21:40:19,311 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:19,311 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-31 21:40:19,311 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:19,311 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-31 21:40:19,311 INFO    SenderThread:1748203 [file_pusher.py:join():175] waiting for file pusher
+2024-12-31 21:40:19,314 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 12
+2024-12-31 21:40:19,314 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:19,314 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-31 21:40:19,314 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:19,314 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-31 21:40:19,315 INFO    SenderThread:1748203 [file_stream.py:finish():601] file stream finish called
+2024-12-31 21:40:19,504 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:19,533 INFO    SenderThread:1748203 [file_stream.py:finish():605] file stream finish is done
+2024-12-31 21:40:19,533 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 13
+2024-12-31 21:40:19,533 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:19,533 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:19,533 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-31 21:40:19,533 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:19,533 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-31 21:40:19,534 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 14
+2024-12-31 21:40:19,534 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:19,534 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-31 21:40:19,534 DEBUG   SenderThread:1748203 [sender.py:send():379] send: final
+2024-12-31 21:40:19,534 DEBUG   SenderThread:1748203 [sender.py:send():379] send: footer
+2024-12-31 21:40:19,535 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:19,535 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-31 21:40:19,535 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:19,535 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: server_info
+2024-12-31 21:40:19,535 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:19,535 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-31 21:40:19,536 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-31 21:40:19,536 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:19,537 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:19,537 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: server_info
+2024-12-31 21:40:19,603 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:19,604 INFO    MainThread:1748203 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-31 21:40:19,604 INFO    MainThread:1748203 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-31 21:40:19,604 INFO    MainThread:1748203 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-31 21:40:19,604 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-31 21:40:19,605 INFO    HandlerThread:1748203 [handler.py:finish():882] shutting down handler
+2024-12-31 21:40:20,537 INFO    WriterThread:1748203 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/run-c7zbixne.wandb
+2024-12-31 21:40:20,604 INFO    SenderThread:1748203 [sender.py:finish():1615] shutting down sender
+2024-12-31 21:40:20,604 INFO    SenderThread:1748203 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 21:40:20,604 INFO    SenderThread:1748203 [file_pusher.py:join():175] waiting for file pusher
diff --git a/c7zbixne/wandb/debug.log b/c7zbixne/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..f3238057c2f75457e753c19e4fc0cbdec21b5efd
--- /dev/null
+++ b/c7zbixne/wandb/debug.log
@@ -0,0 +1,31 @@
+2024-12-31 21:35:03,436 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Configure stats pid to 1748029
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/settings
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/logs/debug.log
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/logs/debug-internal.log
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:init():569] calling init triggers
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne'}
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:init():619] starting backend
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:init():623] setting up manager
+2024-12-31 21:35:03,441 INFO    MainThread:1748029 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-31 21:35:03,441 INFO    MainThread:1748029 [wandb_init.py:init():631] backend started and connected
+2024-12-31 21:35:03,453 INFO    MainThread:1748029 [wandb_init.py:init():720] updated telemetry
+2024-12-31 21:35:03,461 INFO    MainThread:1748029 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-31 21:35:03,701 INFO    MainThread:1748029 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-31 21:35:03,770 INFO    MainThread:1748029 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-31 21:35:03,771 INFO    MainThread:1748029 [wandb_init.py:init():804] starting run threads in backend
+2024-12-31 21:35:04,121 INFO    MainThread:1748029 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-31 21:35:04,121 INFO    MainThread:1748029 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-31 21:35:04,121 INFO    MainThread:1748029 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-31 21:35:04,122 INFO    MainThread:1748029 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-31 21:35:04,123 INFO    MainThread:1748029 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-31 21:35:04,125 INFO    MainThread:1748029 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/tb, True
+2024-12-31 21:35:18,758 INFO    MainThread:1748029 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2024-12-31 21:40:20,605 WARNING MsgRouterThr:1748029 [router.py:message_loop():77] message_loop has been closed
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code/FISH/eval_robot.py b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..41791869dc3657b820d72a52a39ee14580eb36bd
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				"config.grasped_obj_name": grasped_obj_name,
+				"config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					total_num_episodes += filtered_run.summary_metrics['episode']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..64bcc24202d74ca13882937ecba1c211bbb8d26a
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1006_0'
+wandb_notes:
+  desc: null
+  value: 1006_0_req_1352_0restarted_2
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: true
+use_sdf_maps:
+  desc: null
+  value: true
+use_normals_maps:
+  desc: null
+  value: true
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: true
+        use_sdf_maps: true
+        use_normals_maps: true
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: true
+          use_sdf_maps: true
+          use_normals_maps: true
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1735698903
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: greece
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/diff.patch b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..e09588e1c3357af1242f50ab4ab556d71e342fdb
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/diff.patch
@@ -0,0 +1,33 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 0f6a9ec..c76321b 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -133,7 +133,10 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
+-wandb_run_id: '1007_0' 
++# wandb_run_id: '1007_0' 
++
++# RGBD+mask+contact(all ftrs)+act history
++wandb_run_id: '1006_0'
+ 
+ 
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 21e335c..fe71904 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -155,7 +155,11 @@ run_id_list = [
+     # '1022_0',
+     # '1023_0',
+     # '1007_0',
+-    '1007_0'
++    # '1007_0'
++]
++
++run_id_list = [
++    '1006_0',
+ ]
+ 
+ checkpoint_epoch = 12000
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..c30a13305f74325cdb6b6fee6f57b4b907700d90
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.12855625160016762, 0.9510095416597523, 0.1475618631352536, 0.9175734370111737, 0.25050920253021436, 0.8823125510643227, 0.3999994695386616, 0.8205093293741077]]}
\ No newline at end of file
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..a753f10562d6fdb49a38b517f1e4d3ce1594e56b
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[5, 10, 0.5, 0.16378831870127553, 0.8324364423086958, 0.22730267053336026, 0.8105924724911346, 0.2652893662922119, 0.7343519180544403, 0.279645524963104, 0.7253974377658183]]}
\ No newline at end of file
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4 b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1606c79932554d899f77ab0de41f643ec5bad6b5
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8a408e3df7bfc74cc89f4bb1e1a72dba36827787ddaf45bcdc22e92b20b160b
+size 1465413
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4 b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9ba595fe3a153cf73d7f9f43206af10791b34879
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb902dfcfac85de572711b69b5169f14cd46b193216b8d0fb49f7108c7df956a
+size 609556
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4 b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..6423327b771a47e803c87a90aca4f8b8292d2927
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dea02de20bdd586cd5fdc45e12e91aa0201c0f251b47214c0c207f72e26d4bbf
+size 1059657
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4 b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..70dda4306f1be91d1e747f4eadd35d0847dc4321
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d10ed98fe5d597fa5a89e7f68b60f25182c82904b1d424cc5d6328ea83717bda
+size 1466419
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4 b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bb3721983cf3b167e0829a2368557500e6f8c76f
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:578f39c6eeca4936625ce4639b6f651b2683d682a9c827e509298a184631d901
+size 1516972
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..aa09bfb8889dcad11dc1eb97bbd9a9bcd5302eae
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
@@ -0,0 +1,5109 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1735698918.759573]: resetting environment
+[INFO] [1735698918.765497]: cleared current plan
+[INFO] [1735698918.766621]: moving to home
+[INFO] [1735698919.769216]: reached home
+[INFO] [1735698919.770474]: reset action history
+[INFO] [1735698922.482240]: environment reset
+Starting episode 0
+[INFO] [1735698922.487072]: resetting environment
+[INFO] [1735698922.491613]: cleared current plan
+[INFO] [1735698922.492692]: moving to home
+[INFO] [1735698923.495755]: reached home
+[INFO] [1735698923.497030]: reset action history
+[INFO] [1735698926.256861]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1735698930.567045]: resetting environment
+[INFO] [1735698930.574116]: cleared current plan
+[INFO] [1735698930.575464]: moving to home
+[INFO] [1735698931.577475]: reached home
+[INFO] [1735698931.579159]: reset action history
+[INFO] [1735698934.288113]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/episode_rosbags/episode_0_2024-12-31-21-35-34.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1748029.log
+started roslaunch server http://158.130.50.37:43965/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [1748332]
+process[print_text-2]: started with pid [1748333]
+process[rosbag_record-3]: started with pid [1748334]
+started bagging!
+[INFO] [1735698935.060235]: received mask_obs with header seq: 2227
+stamp:
+  secs: 1735698935
+  nsecs:  63365459
+frame_id: '' and grasped_object_mask header seq: 187205
+stamp:
+  secs: 1735698934
+  nsecs: 817654133
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698935.326387]: received mask_obs with header seq: 2228
+stamp:
+  secs: 1735698935
+  nsecs: 329129934
+frame_id: '' and grasped_object_mask header seq: 187214
+stamp:
+  secs: 1735698935
+  nsecs: 117411375
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698935.553972]: received mask_obs with header seq: 2229
+stamp:
+  secs: 1735698935
+  nsecs: 557550907
+frame_id: '' and grasped_object_mask header seq: 187221
+stamp:
+  secs: 1735698935
+  nsecs: 350556612
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698935.792409]: received mask_obs with header seq: 2230
+stamp:
+  secs: 1735698935
+  nsecs: 796054124
+frame_id: '' and grasped_object_mask header seq: 187228
+stamp:
+  secs: 1735698935
+  nsecs: 583699465
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698936.036521]: received mask_obs with header seq: 2231
+stamp:
+  secs: 1735698936
+  nsecs:  39746761
+frame_id: '' and grasped_object_mask header seq: 187235
+stamp:
+  secs: 1735698935
+  nsecs: 816842318
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698936.269264]: received mask_obs with header seq: 2232
+stamp:
+  secs: 1735698936
+  nsecs: 272879838
+frame_id: '' and grasped_object_mask header seq: 187242
+stamp:
+  secs: 1735698936
+  nsecs:  49983978
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698936.534998]: received mask_obs with header seq: 2233
+stamp:
+  secs: 1735698936
+  nsecs: 538572311
+frame_id: '' and grasped_object_mask header seq: 187250
+stamp:
+  secs: 1735698936
+  nsecs: 316431522
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698936.794269]: received mask_obs with header seq: 2234
+stamp:
+  secs: 1735698936
+  nsecs: 797788619
+frame_id: '' and grasped_object_mask header seq: 187257
+stamp:
+  secs: 1735698936
+  nsecs: 549580812
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698937.051877]: received mask_obs with header seq: 2235
+stamp:
+  secs: 1735698937
+  nsecs:  55022001
+frame_id: '' and grasped_object_mask header seq: 187265
+stamp:
+  secs: 1735698936
+  nsecs: 816038132
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698937.311771]: received mask_obs with header seq: 2236
+stamp:
+  secs: 1735698937
+  nsecs: 315345764
+frame_id: '' and grasped_object_mask header seq: 187273
+stamp:
+  secs: 1735698937
+  nsecs:  82496881
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698937.573049]: received mask_obs with header seq: 2237
+stamp:
+  secs: 1735698937
+  nsecs: 576621055
+frame_id: '' and grasped_object_mask header seq: 187281
+stamp:
+  secs: 1735698937
+  nsecs: 348954916
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698937.824502]: received mask_obs with header seq: 2238
+stamp:
+  secs: 1735698937
+  nsecs: 828063488
+frame_id: '' and grasped_object_mask header seq: 187288
+stamp:
+  secs: 1735698937
+  nsecs: 582093477
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698938.085009]: received mask_obs with header seq: 2239
+stamp:
+  secs: 1735698938
+  nsecs:  88630676
+frame_id: '' and grasped_object_mask header seq: 187296
+stamp:
+  secs: 1735698937
+  nsecs: 848538637
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698938.352543]: received mask_obs with header seq: 2240
+stamp:
+  secs: 1735698938
+  nsecs: 356075048
+frame_id: '' and grasped_object_mask header seq: 187304
+stamp:
+  secs: 1735698938
+  nsecs: 114982128
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698938.609961]: received mask_obs with header seq: 2241
+stamp:
+  secs: 1735698938
+  nsecs: 613559961
+frame_id: '' and grasped_object_mask header seq: 187312
+stamp:
+  secs: 1735698938
+  nsecs: 381425142
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698938.878425]: received mask_obs with header seq: 2242
+stamp:
+  secs: 1735698938
+  nsecs: 882084369
+frame_id: '' and grasped_object_mask header seq: 187320
+stamp:
+  secs: 1735698938
+  nsecs: 647864580
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698939.143374]: received mask_obs with header seq: 2243
+stamp:
+  secs: 1735698939
+  nsecs: 146993637
+frame_id: '' and grasped_object_mask header seq: 187328
+stamp:
+  secs: 1735698938
+  nsecs: 914304256
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698939.402340]: received mask_obs with header seq: 2244
+stamp:
+  secs: 1735698939
+  nsecs: 405972719
+frame_id: '' and grasped_object_mask header seq: 187336
+stamp:
+  secs: 1735698939
+  nsecs: 180742502
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is 106148340 for nearest: 1735698939520026781 - target: 1735698939413878441 at idx 76
+[INFO] [1735698939.671342]: received mask_obs with header seq: 2245
+stamp:
+  secs: 1735698939
+  nsecs: 675010919
+frame_id: '' and grasped_object_mask header seq: 187343
+stamp:
+  secs: 1735698939
+  nsecs: 413878441
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698939.937423]: received mask_obs with header seq: 2246
+stamp:
+  secs: 1735698939
+  nsecs: 941077232
+frame_id: '' and grasped_object_mask header seq: 187352
+stamp:
+  secs: 1735698939
+  nsecs: 713638544
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698940.188372]: received mask_obs with header seq: 2247
+stamp:
+  secs: 1735698940
+  nsecs: 191965103
+frame_id: '' and grasped_object_mask header seq: 187359
+stamp:
+  secs: 1735698939
+  nsecs: 946785212
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698940.451069]: received mask_obs with header seq: 2248
+stamp:
+  secs: 1735698940
+  nsecs: 454725027
+frame_id: '' and grasped_object_mask header seq: 187367
+stamp:
+  secs: 1735698940
+  nsecs: 213237762
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698940.721467]: received mask_obs with header seq: 2249
+stamp:
+  secs: 1735698940
+  nsecs: 725078821
+frame_id: '' and grasped_object_mask header seq: 187375
+stamp:
+  secs: 1735698940
+  nsecs: 479692221
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698940.979685]: received mask_obs with header seq: 2250
+stamp:
+  secs: 1735698940
+  nsecs: 982393264
+frame_id: '' and grasped_object_mask header seq: 187383
+stamp:
+  secs: 1735698940
+  nsecs: 746149063
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698941.241487]: received mask_obs with header seq: 2251
+stamp:
+  secs: 1735698941
+  nsecs: 244812250
+frame_id: '' and grasped_object_mask header seq: 187391
+stamp:
+  secs: 1735698941
+  nsecs:  12605906
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698941.507297]: received mask_obs with header seq: 2252
+stamp:
+  secs: 1735698941
+  nsecs: 510482072
+frame_id: '' and grasped_object_mask header seq: 187399
+stamp:
+  secs: 1735698941
+  nsecs: 279062986
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698941.773157]: received mask_obs with header seq: 2253
+stamp:
+  secs: 1735698941
+  nsecs: 776725292
+frame_id: '' and grasped_object_mask header seq: 187407
+stamp:
+  secs: 1735698941
+  nsecs: 545520306
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698942.029115]: received mask_obs with header seq: 2254
+stamp:
+  secs: 1735698942
+  nsecs:  31357288
+frame_id: '' and grasped_object_mask header seq: 187414
+stamp:
+  secs: 1735698941
+  nsecs: 778671026
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698942.284937]: received mask_obs with header seq: 2255
+stamp:
+  secs: 1735698942
+  nsecs: 288524866
+frame_id: '' and grasped_object_mask header seq: 187422
+stamp:
+  secs: 1735698942
+  nsecs:  45128822
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698942.539626]: received mask_obs with header seq: 2256
+stamp:
+  secs: 1735698942
+  nsecs: 543215274
+frame_id: '' and grasped_object_mask header seq: 187430
+stamp:
+  secs: 1735698942
+  nsecs: 311587095
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698942.806289]: received mask_obs with header seq: 2257
+stamp:
+  secs: 1735698942
+  nsecs: 809574365
+frame_id: '' and grasped_object_mask header seq: 187438
+stamp:
+  secs: 1735698942
+  nsecs: 578039646
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698943.070801]: received mask_obs with header seq: 2258
+stamp:
+  secs: 1735698943
+  nsecs:  74387550
+frame_id: '' and grasped_object_mask header seq: 187446
+stamp:
+  secs: 1735698942
+  nsecs: 844492197
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698943.317775]: received mask_obs with header seq: 2259
+stamp:
+  secs: 1735698943
+  nsecs: 321138858
+frame_id: '' and grasped_object_mask header seq: 187453
+stamp:
+  secs: 1735698943
+  nsecs:  77636957
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698943.584033]: received mask_obs with header seq: 2260
+stamp:
+  secs: 1735698943
+  nsecs: 587163925
+frame_id: '' and grasped_object_mask header seq: 187461
+stamp:
+  secs: 1735698943
+  nsecs: 344089508
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698943.839341]: received mask_obs with header seq: 2261
+stamp:
+  secs: 1735698943
+  nsecs: 842746257
+frame_id: '' and grasped_object_mask header seq: 187469
+stamp:
+  secs: 1735698943
+  nsecs: 610546589
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698944.106613]: received mask_obs with header seq: 2262
+stamp:
+  secs: 1735698944
+  nsecs: 110221147
+frame_id: '' and grasped_object_mask header seq: 187477
+stamp:
+  secs: 1735698943
+  nsecs: 877004147
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698944.353690]: received mask_obs with header seq: 2263
+stamp:
+  secs: 1735698944
+  nsecs: 356820583
+frame_id: '' and grasped_object_mask header seq: 187484
+stamp:
+  secs: 1735698944
+  nsecs: 110155344
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698944.616795]: received mask_obs with header seq: 2264
+stamp:
+  secs: 1735698944
+  nsecs: 620119571
+frame_id: '' and grasped_object_mask header seq: 187492
+stamp:
+  secs: 1735698944
+  nsecs: 376613140
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698944.872511]: received mask_obs with header seq: 2265
+stamp:
+  secs: 1735698944
+  nsecs: 875631093
+frame_id: '' and grasped_object_mask header seq: 187500
+stamp:
+  secs: 1735698944
+  nsecs: 643065214
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698945.138429]: received mask_obs with header seq: 2266
+stamp:
+  secs: 1735698945
+  nsecs: 141919851
+frame_id: '' and grasped_object_mask header seq: 187508
+stamp:
+  secs: 1735698944
+  nsecs: 909517527
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698945.388826]: received mask_obs with header seq: 2267
+stamp:
+  secs: 1735698945
+  nsecs: 392397642
+frame_id: '' and grasped_object_mask header seq: 187515
+stamp:
+  secs: 1735698945
+  nsecs: 142663956
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is 110911808 for nearest: 1735698945520028315 - target: 1735698945409116507 at idx 158
+[INFO] [1735698945.652912]: received mask_obs with header seq: 2268
+stamp:
+  secs: 1735698945
+  nsecs: 656563043
+frame_id: '' and grasped_object_mask header seq: 187523
+stamp:
+  secs: 1735698945
+  nsecs: 409116507
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698945.908086]: received mask_obs with header seq: 2269
+stamp:
+  secs: 1735698945
+  nsecs: 911465644
+frame_id: '' and grasped_object_mask header seq: 187531
+stamp:
+  secs: 1735698945
+  nsecs: 678576708
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698946.171488]: received mask_obs with header seq: 2270
+stamp:
+  secs: 1735698946
+  nsecs: 174108743
+frame_id: '' and grasped_object_mask header seq: 187539
+stamp:
+  secs: 1735698945
+  nsecs: 948290586
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698946.434936]: received mask_obs with header seq: 2271
+stamp:
+  secs: 1735698946
+  nsecs: 438315629
+frame_id: '' and grasped_object_mask header seq: 187547
+stamp:
+  secs: 1735698946
+  nsecs: 218159199
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698946.684844]: received mask_obs with header seq: 2272
+stamp:
+  secs: 1735698946
+  nsecs: 688464641
+frame_id: '' and grasped_object_mask header seq: 187554
+stamp:
+  secs: 1735698946
+  nsecs: 454065084
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698946.955266]: received mask_obs with header seq: 2273
+stamp:
+  secs: 1735698946
+  nsecs: 958878517
+frame_id: '' and grasped_object_mask header seq: 187562
+stamp:
+  secs: 1735698946
+  nsecs: 720213652
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698947.227415]: received mask_obs with header seq: 2274
+stamp:
+  secs: 1735698947
+  nsecs: 229952812
+frame_id: '' and grasped_object_mask header seq: 187570
+stamp:
+  secs: 1735698946
+  nsecs: 986341238
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698947.490617]: received mask_obs with header seq: 2275
+stamp:
+  secs: 1735698947
+  nsecs: 494022130
+frame_id: '' and grasped_object_mask header seq: 187578
+stamp:
+  secs: 1735698947
+  nsecs: 252448082
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698947.750057]: received mask_obs with header seq: 2276
+stamp:
+  secs: 1735698947
+  nsecs: 753552675
+frame_id: '' and grasped_object_mask header seq: 187586
+stamp:
+  secs: 1735698947
+  nsecs: 518556356
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698948.014071]: received mask_obs with header seq: 2277
+stamp:
+  secs: 1735698948
+  nsecs:  17651796
+frame_id: '' and grasped_object_mask header seq: 187594
+stamp:
+  secs: 1735698947
+  nsecs: 784688234
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698948.277995]: received mask_obs with header seq: 2278
+stamp:
+  secs: 1735698948
+  nsecs: 281486511
+frame_id: '' and grasped_object_mask header seq: 187602
+stamp:
+  secs: 1735698948
+  nsecs:  50797701
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698948.545893]: received mask_obs with header seq: 2279
+stamp:
+  secs: 1735698948
+  nsecs: 548754453
+frame_id: '' and grasped_object_mask header seq: 187610
+stamp:
+  secs: 1735698948
+  nsecs: 316885710
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698948.806364]: received mask_obs with header seq: 2280
+stamp:
+  secs: 1735698948
+  nsecs: 809996843
+frame_id: '' and grasped_object_mask header seq: 187618
+stamp:
+  secs: 1735698948
+  nsecs: 582994938
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698949.071418]: received mask_obs with header seq: 2281
+stamp:
+  secs: 1735698949
+  nsecs:  75087785
+frame_id: '' and grasped_object_mask header seq: 187626
+stamp:
+  secs: 1735698948
+  nsecs: 849112988
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698949.334185]: received mask_obs with header seq: 2282
+stamp:
+  secs: 1735698949
+  nsecs: 337822914
+frame_id: '' and grasped_object_mask header seq: 187633
+stamp:
+  secs: 1735698949
+  nsecs:  81947327
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698949.604457]: received mask_obs with header seq: 2283
+stamp:
+  secs: 1735698949
+  nsecs: 608089685
+frame_id: '' and grasped_object_mask header seq: 187641
+stamp:
+  secs: 1735698949
+  nsecs: 348023415
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698949.865529]: received mask_obs with header seq: 2284
+stamp:
+  secs: 1735698949
+  nsecs: 868360280
+frame_id: '' and grasped_object_mask header seq: 187650
+stamp:
+  secs: 1735698949
+  nsecs: 647414207
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698950.116980]: received mask_obs with header seq: 2285
+stamp:
+  secs: 1735698950
+  nsecs: 120542049
+frame_id: '' and grasped_object_mask header seq: 187657
+stamp:
+  secs: 1735698949
+  nsecs: 880277395
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698950.374388]: received mask_obs with header seq: 2286
+stamp:
+  secs: 1735698950
+  nsecs: 377881765
+frame_id: '' and grasped_object_mask header seq: 187665
+stamp:
+  secs: 1735698950
+  nsecs: 146386147
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698950.638424]: received mask_obs with header seq: 2287
+stamp:
+  secs: 1735698950
+  nsecs: 642076492
+frame_id: '' and grasped_object_mask header seq: 187673
+stamp:
+  secs: 1735698950
+  nsecs: 412474394
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698950.900786]: received mask_obs with header seq: 2288
+stamp:
+  secs: 1735698950
+  nsecs: 904191255
+frame_id: '' and grasped_object_mask header seq: 187681
+stamp:
+  secs: 1735698950
+  nsecs: 678628206
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698951.152069]: received mask_obs with header seq: 2289
+stamp:
+  secs: 1735698951
+  nsecs: 155627965
+frame_id: '' and grasped_object_mask header seq: 187688
+stamp:
+  secs: 1735698950
+  nsecs: 911497593
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698951.407521]: received mask_obs with header seq: 2290
+stamp:
+  secs: 1735698951
+  nsecs: 410742759
+frame_id: '' and grasped_object_mask header seq: 187696
+stamp:
+  secs: 1735698951
+  nsecs: 177615881
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698951.672498]: received mask_obs with header seq: 2291
+stamp:
+  secs: 1735698951
+  nsecs: 676119327
+frame_id: '' and grasped_object_mask header seq: 187704
+stamp:
+  secs: 1735698951
+  nsecs: 443719625
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698951.935873]: received mask_obs with header seq: 2292
+stamp:
+  secs: 1735698951
+  nsecs: 939511060
+frame_id: '' and grasped_object_mask header seq: 187712
+stamp:
+  secs: 1735698951
+  nsecs: 709858894
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698952.196929]: received mask_obs with header seq: 2293
+stamp:
+  secs: 1735698952
+  nsecs: 200585365
+frame_id: '' and grasped_object_mask header seq: 187719
+stamp:
+  secs: 1735698951
+  nsecs: 942713737
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698952.455139]: received mask_obs with header seq: 2294
+stamp:
+  secs: 1735698952
+  nsecs: 458661556
+frame_id: '' and grasped_object_mask header seq: 187727
+stamp:
+  secs: 1735698952
+  nsecs: 208812952
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698952.713140]: received mask_obs with header seq: 2295
+stamp:
+  secs: 1735698952
+  nsecs: 716733932
+frame_id: '' and grasped_object_mask header seq: 187735
+stamp:
+  secs: 1735698952
+  nsecs: 474908590
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698952.976033]: received mask_obs with header seq: 2296
+stamp:
+  secs: 1735698952
+  nsecs: 979652404
+frame_id: '' and grasped_object_mask header seq: 187743
+stamp:
+  secs: 1735698952
+  nsecs: 741050959
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698953.236821]: received mask_obs with header seq: 2297
+stamp:
+  secs: 1735698953
+  nsecs: 240442991
+frame_id: '' and grasped_object_mask header seq: 187751
+stamp:
+  secs: 1735698953
+  nsecs:   7172823
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698953.499888]: received mask_obs with header seq: 2298
+stamp:
+  secs: 1735698953
+  nsecs: 503444433
+frame_id: '' and grasped_object_mask header seq: 187759
+stamp:
+  secs: 1735698953
+  nsecs: 273274660
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698953.755080]: received mask_obs with header seq: 2299
+stamp:
+  secs: 1735698953
+  nsecs: 758688449
+frame_id: '' and grasped_object_mask header seq: 187766
+stamp:
+  secs: 1735698953
+  nsecs: 506121635
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698954.026726]: received mask_obs with header seq: 2300
+stamp:
+  secs: 1735698954
+  nsecs:  29056549
+frame_id: '' and grasped_object_mask header seq: 187774
+stamp:
+  secs: 1735698953
+  nsecs: 772264481
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698954.279537]: received mask_obs with header seq: 2301
+stamp:
+  secs: 1735698954
+  nsecs: 282831668
+frame_id: '' and grasped_object_mask header seq: 187782
+stamp:
+  secs: 1735698954
+  nsecs:  38385391
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698954.538395]: received mask_obs with header seq: 2302
+stamp:
+  secs: 1735698954
+  nsecs: 541898012
+frame_id: '' and grasped_object_mask header seq: 187790
+stamp:
+  secs: 1735698954
+  nsecs: 304485559
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698954.802460]: received mask_obs with header seq: 2303
+stamp:
+  secs: 1735698954
+  nsecs: 805984020
+frame_id: '' and grasped_object_mask header seq: 187798
+stamp:
+  secs: 1735698954
+  nsecs: 570601940
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698955.068942]: received mask_obs with header seq: 2304
+stamp:
+  secs: 1735698955
+  nsecs:  72324037
+frame_id: '' and grasped_object_mask header seq: 187806
+stamp:
+  secs: 1735698954
+  nsecs: 836724281
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698955.320956]: received mask_obs with header seq: 2305
+stamp:
+  secs: 1735698955
+  nsecs: 324586391
+frame_id: '' and grasped_object_mask header seq: 187813
+stamp:
+  secs: 1735698955
+  nsecs:  69563866
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698955.578897]: received mask_obs with header seq: 2306
+stamp:
+  secs: 1735698955
+  nsecs: 582534074
+frame_id: '' and grasped_object_mask header seq: 187821
+stamp:
+  secs: 1735698955
+  nsecs: 335645676
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698955.840767]: received mask_obs with header seq: 2307
+stamp:
+  secs: 1735698955
+  nsecs: 844228506
+frame_id: '' and grasped_object_mask header seq: 187829
+stamp:
+  secs: 1735698955
+  nsecs: 601776361
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698956.104043]: received mask_obs with header seq: 2308
+stamp:
+  secs: 1735698956
+  nsecs: 107251882
+frame_id: '' and grasped_object_mask header seq: 187837
+stamp:
+  secs: 1735698955
+  nsecs: 867915869
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698956.367812]: received mask_obs with header seq: 2309
+stamp:
+  secs: 1735698956
+  nsecs: 371453046
+frame_id: '' and grasped_object_mask header seq: 187845
+stamp:
+  secs: 1735698956
+  nsecs: 134034634
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698956.619493]: received mask_obs with header seq: 2310
+stamp:
+  secs: 1735698956
+  nsecs: 623086929
+frame_id: '' and grasped_object_mask header seq: 187852
+stamp:
+  secs: 1735698956
+  nsecs: 366872311
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698956.873773]: received mask_obs with header seq: 2311
+stamp:
+  secs: 1735698956
+  nsecs: 877421617
+frame_id: '' and grasped_object_mask header seq: 187860
+stamp:
+  secs: 1735698956
+  nsecs: 633036137
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698957.138530]: received mask_obs with header seq: 2312
+stamp:
+  secs: 1735698957
+  nsecs: 142027378
+frame_id: '' and grasped_object_mask header seq: 187868
+stamp:
+  secs: 1735698956
+  nsecs: 899198532
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698957.401498]: received mask_obs with header seq: 2313
+stamp:
+  secs: 1735698957
+  nsecs: 405135869
+frame_id: '' and grasped_object_mask header seq: 187876
+stamp:
+  secs: 1735698957
+  nsecs: 165341616
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is 121824490 for nearest: 1735698957520025956 - target: 1735698957398201466 at idx 249
+[INFO] [1735698957.658485]: received mask_obs with header seq: 2314
+stamp:
+  secs: 1735698957
+  nsecs: 661936998
+frame_id: '' and grasped_object_mask header seq: 187883
+stamp:
+  secs: 1735698957
+  nsecs: 398201466
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698957.935756]: received mask_obs with header seq: 2315
+stamp:
+  secs: 1735698957
+  nsecs: 938468933
+frame_id: '' and grasped_object_mask header seq: 187891
+stamp:
+  secs: 1735698957
+  nsecs: 664350510
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698958.203985]: received mask_obs with header seq: 2316
+stamp:
+  secs: 1735698958
+  nsecs: 207409620
+frame_id: '' and grasped_object_mask header seq: 187900
+stamp:
+  secs: 1735698957
+  nsecs: 963747025
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698958.469236]: received mask_obs with header seq: 2317
+stamp:
+  secs: 1735698958
+  nsecs: 472029447
+frame_id: '' and grasped_object_mask header seq: 187908
+stamp:
+  secs: 1735698958
+  nsecs: 229854822
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698958.738001]: received mask_obs with header seq: 2318
+stamp:
+  secs: 1735698958
+  nsecs: 741633653
+frame_id: '' and grasped_object_mask header seq: 187916
+stamp:
+  secs: 1735698958
+  nsecs: 495963573
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698959.002804]: received mask_obs with header seq: 2319
+stamp:
+  secs: 1735698959
+  nsecs:   6055593
+frame_id: '' and grasped_object_mask header seq: 187924
+stamp:
+  secs: 1735698958
+  nsecs: 762099028
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698959.263835]: received mask_obs with header seq: 2320
+stamp:
+  secs: 1735698959
+  nsecs: 267189502
+frame_id: '' and grasped_object_mask header seq: 187932
+stamp:
+  secs: 1735698959
+  nsecs:  28212309
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698959.536351]: received mask_obs with header seq: 2321
+stamp:
+  secs: 1735698959
+  nsecs: 539864540
+frame_id: '' and grasped_object_mask header seq: 187939
+stamp:
+  secs: 1735698959
+  nsecs: 261044502
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698959.794585]: received mask_obs with header seq: 2322
+stamp:
+  secs: 1735698959
+  nsecs: 798256635
+frame_id: '' and grasped_object_mask header seq: 187947
+stamp:
+  secs: 1735698959
+  nsecs: 527156830
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698960.044745]: received mask_obs with header seq: 2323
+stamp:
+  secs: 1735698960
+  nsecs:  48365116
+frame_id: '' and grasped_object_mask header seq: 187955
+stamp:
+  secs: 1735698959
+  nsecs: 793297291
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698960.305791]: received mask_obs with header seq: 2324
+stamp:
+  secs: 1735698960
+  nsecs: 309420824
+frame_id: '' and grasped_object_mask header seq: 187963
+stamp:
+  secs: 1735698960
+  nsecs:  59417486
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698960.572753]: received mask_obs with header seq: 2325
+stamp:
+  secs: 1735698960
+  nsecs: 576047658
+frame_id: '' and grasped_object_mask header seq: 187971
+stamp:
+  secs: 1735698960
+  nsecs: 325516701
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698960.839457]: received mask_obs with header seq: 2326
+stamp:
+  secs: 1735698960
+  nsecs: 842941522
+frame_id: '' and grasped_object_mask header seq: 187979
+stamp:
+  secs: 1735698960
+  nsecs: 591650009
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698961.116399]: received mask_obs with header seq: 2327
+stamp:
+  secs: 1735698961
+  nsecs: 120021581
+frame_id: '' and grasped_object_mask header seq: 187983
+stamp:
+  secs: 1735698960
+  nsecs: 724720478
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698961.382403]: received mask_obs with header seq: 2328
+stamp:
+  secs: 1735698961
+  nsecs: 385959148
+frame_id: '' and grasped_object_mask header seq: 187995
+stamp:
+  secs: 1735698961
+  nsecs: 123900652
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698961.634213]: received mask_obs with header seq: 2329
+stamp:
+  secs: 1735698961
+  nsecs: 637378454
+frame_id: '' and grasped_object_mask header seq: 188003
+stamp:
+  secs: 1735698961
+  nsecs: 389994383
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698961.888487]: received mask_obs with header seq: 2330
+stamp:
+  secs: 1735698961
+  nsecs: 892147302
+frame_id: '' and grasped_object_mask header seq: 188011
+stamp:
+  secs: 1735698961
+  nsecs: 657769442
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698962.159775]: received mask_obs with header seq: 2331
+stamp:
+  secs: 1735698962
+  nsecs: 163420438
+frame_id: '' and grasped_object_mask header seq: 188019
+stamp:
+  secs: 1735698961
+  nsecs: 925812960
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698962.421365]: received mask_obs with header seq: 2332
+stamp:
+  secs: 1735698962
+  nsecs: 424565315
+frame_id: '' and grasped_object_mask header seq: 188027
+stamp:
+  secs: 1735698962
+  nsecs: 194011927
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698962.686681]: received mask_obs with header seq: 2333
+stamp:
+  secs: 1735698962
+  nsecs: 690093040
+frame_id: '' and grasped_object_mask header seq: 188035
+stamp:
+  secs: 1735698962
+  nsecs: 462007046
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698962.953421]: received mask_obs with header seq: 2334
+stamp:
+  secs: 1735698962
+  nsecs: 957040548
+frame_id: '' and grasped_object_mask header seq: 188042
+stamp:
+  secs: 1735698962
+  nsecs: 695162058
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698963.219501]: received mask_obs with header seq: 2335
+stamp:
+  secs: 1735698963
+  nsecs: 222946643
+frame_id: '' and grasped_object_mask header seq: 188051
+stamp:
+  secs: 1735698962
+  nsecs: 994933367
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698963.470165]: received mask_obs with header seq: 2336
+stamp:
+  secs: 1735698963
+  nsecs: 473835468
+frame_id: '' and grasped_object_mask header seq: 188058
+stamp:
+  secs: 1735698963
+  nsecs: 228088617
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698963.737133]: received mask_obs with header seq: 2337
+stamp:
+  secs: 1735698963
+  nsecs: 740759611
+frame_id: '' and grasped_object_mask header seq: 188066
+stamp:
+  secs: 1735698963
+  nsecs: 494548798
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698963.993793]: received mask_obs with header seq: 2338
+stamp:
+  secs: 1735698963
+  nsecs: 997461318
+frame_id: '' and grasped_object_mask header seq: 188074
+stamp:
+  secs: 1735698963
+  nsecs: 760999680
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698964.253411]: received mask_obs with header seq: 2339
+stamp:
+  secs: 1735698964
+  nsecs: 257025241
+frame_id: '' and grasped_object_mask header seq: 188082
+stamp:
+  secs: 1735698964
+  nsecs:  27449131
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698964.517978]: received mask_obs with header seq: 2340
+stamp:
+  secs: 1735698964
+  nsecs: 521589517
+frame_id: '' and grasped_object_mask header seq: 188090
+stamp:
+  secs: 1735698964
+  nsecs: 293898582
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698964.786910]: received mask_obs with header seq: 2341
+stamp:
+  secs: 1735698964
+  nsecs: 790453195
+frame_id: '' and grasped_object_mask header seq: 188097
+stamp:
+  secs: 1735698964
+  nsecs: 527041912
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698965.051484]: received mask_obs with header seq: 2342
+stamp:
+  secs: 1735698965
+  nsecs:  55109739
+frame_id: '' and grasped_object_mask header seq: 188106
+stamp:
+  secs: 1735698964
+  nsecs: 826796532
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698965.315340]: received mask_obs with header seq: 2343
+stamp:
+  secs: 1735698965
+  nsecs: 318807125
+frame_id: '' and grasped_object_mask header seq: 188114
+stamp:
+  secs: 1735698965
+  nsecs:  93244314
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698965.569629]: received mask_obs with header seq: 2344
+stamp:
+  secs: 1735698965
+  nsecs: 573127746
+frame_id: '' and grasped_object_mask header seq: 188121
+stamp:
+  secs: 1735698965
+  nsecs: 326385260
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698965.823100]: received mask_obs with header seq: 2345
+stamp:
+  secs: 1735698965
+  nsecs: 826573371
+frame_id: '' and grasped_object_mask header seq: 188129
+stamp:
+  secs: 1735698965
+  nsecs: 592839718
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698966.093770]: received mask_obs with header seq: 2346
+stamp:
+  secs: 1735698966
+  nsecs:  97209692
+frame_id: '' and grasped_object_mask header seq: 188137
+stamp:
+  secs: 1735698965
+  nsecs: 859297037
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698966.353920]: received mask_obs with header seq: 2347
+stamp:
+  secs: 1735698966
+  nsecs: 357585430
+frame_id: '' and grasped_object_mask header seq: 188145
+stamp:
+  secs: 1735698966
+  nsecs: 125754833
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698966.620126]: received mask_obs with header seq: 2348
+stamp:
+  secs: 1735698966
+  nsecs: 623703718
+frame_id: '' and grasped_object_mask header seq: 188153
+stamp:
+  secs: 1735698966
+  nsecs: 392212868
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698966.880277]: received mask_obs with header seq: 2349
+stamp:
+  secs: 1735698966
+  nsecs: 883937835
+frame_id: '' and grasped_object_mask header seq: 188161
+stamp:
+  secs: 1735698966
+  nsecs: 658656836
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698967.135481]: received mask_obs with header seq: 2350
+stamp:
+  secs: 1735698967
+  nsecs: 139004707
+frame_id: '' and grasped_object_mask header seq: 188168
+stamp:
+  secs: 1735698966
+  nsecs: 891791582
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698967.393246]: received mask_obs with header seq: 2351
+stamp:
+  secs: 1735698967
+  nsecs: 396484136
+frame_id: '' and grasped_object_mask header seq: 188176
+stamp:
+  secs: 1735698967
+  nsecs: 158231258
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698967.653784]: received mask_obs with header seq: 2352
+stamp:
+  secs: 1735698967
+  nsecs: 657418727
+frame_id: '' and grasped_object_mask header seq: 188184
+stamp:
+  secs: 1735698967
+  nsecs: 424671888
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698967.917681]: received mask_obs with header seq: 2353
+stamp:
+  secs: 1735698967
+  nsecs: 921286344
+frame_id: '' and grasped_object_mask header seq: 188192
+stamp:
+  secs: 1735698967
+  nsecs: 691109180
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698968.184572]: received mask_obs with header seq: 2354
+stamp:
+  secs: 1735698968
+  nsecs: 188029527
+frame_id: '' and grasped_object_mask header seq: 188200
+stamp:
+  secs: 1735698967
+  nsecs: 957545519
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698968.451951]: received mask_obs with header seq: 2355
+stamp:
+  secs: 1735698968
+  nsecs: 455398797
+frame_id: '' and grasped_object_mask header seq: 188208
+stamp:
+  secs: 1735698968
+  nsecs: 223980904
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698968.719271]: received mask_obs with header seq: 2356
+stamp:
+  secs: 1735698968
+  nsecs: 722908973
+frame_id: '' and grasped_object_mask header seq: 188216
+stamp:
+  secs: 1735698968
+  nsecs: 490420103
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698968.984020]: received mask_obs with header seq: 2357
+stamp:
+  secs: 1735698968
+  nsecs: 987329721
+frame_id: '' and grasped_object_mask header seq: 188224
+stamp:
+  secs: 1735698968
+  nsecs: 756874084
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698969.244976]: received mask_obs with header seq: 2358
+stamp:
+  secs: 1735698969
+  nsecs: 248428106
+frame_id: '' and grasped_object_mask header seq: 188232
+stamp:
+  secs: 1735698969
+  nsecs:  23326874
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698969.345093]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1735698969.499878]: received mask_obs with header seq: 2359
+stamp:
+  secs: 1735698969
+  nsecs: 502983570
+frame_id: '' and grasped_object_mask header seq: 188239
+stamp:
+  secs: 1735698969
+  nsecs: 256473541
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698970.150087]: Stored episode 1.
+[INFO] [1735698970.150803]: Saving video...
+[INFO] [1735698970.361398]: Video saved!
+Starting episode 1
+[INFO] [1735698970.373598]: resetting environment
+[INFO] [1735698970.387833]: cleared current plan
+[INFO] [1735698970.389241]: moving to home
+[INFO] [1735698973.992083]: reached home
+[INFO] [1735698973.992566]: reset action history
+[INFO] [1735698976.704996]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735698992.041944]: resetting environment
+[INFO] [1735698992.045895]: cleared current plan
+[INFO] [1735698992.046072]: moving to home
+[INFO] [1735698993.047508]: reached home
+[INFO] [1735698993.047925]: reset action history
+[INFO] [1735698995.756545]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/episode_rosbags/episode_1_2024-12-31-21-36-35.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1748029.log
+started roslaunch server http://158.130.50.37:38519/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [1748458]
+process[print_text-5]: started with pid [1748482]
+process[rosbag_record-6]: started with pid [1748483]
+started bagging!
+[INFO] [1735698996.494386]: received mask_obs with header seq: 2360
+stamp:
+  secs: 1735698996
+  nsecs: 497819662
+frame_id: '' and grasped_object_mask header seq: 189051
+stamp:
+  secs: 1735698996
+  nsecs: 301328182
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698996.725940]: received mask_obs with header seq: 2361
+stamp:
+  secs: 1735698996
+  nsecs: 729584217
+frame_id: '' and grasped_object_mask header seq: 189057
+stamp:
+  secs: 1735698996
+  nsecs: 501167536
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698996.968168]: received mask_obs with header seq: 2362
+stamp:
+  secs: 1735698996
+  nsecs: 971780300
+frame_id: '' and grasped_object_mask header seq: 189065
+stamp:
+  secs: 1735698996
+  nsecs: 767624140
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698997.206970]: received mask_obs with header seq: 2363
+stamp:
+  secs: 1735698997
+  nsecs: 210372924
+frame_id: '' and grasped_object_mask header seq: 189072
+stamp:
+  secs: 1735698997
+  nsecs:    774384
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698997.447027]: received mask_obs with header seq: 2364
+stamp:
+  secs: 1735698997
+  nsecs: 450484037
+frame_id: '' and grasped_object_mask header seq: 189079
+stamp:
+  secs: 1735698997
+  nsecs: 233923912
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698997.695202]: received mask_obs with header seq: 2365
+stamp:
+  secs: 1735698997
+  nsecs: 698876142
+frame_id: '' and grasped_object_mask header seq: 189086
+stamp:
+  secs: 1735698997
+  nsecs: 467072964
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698997.939991]: received mask_obs with header seq: 2366
+stamp:
+  secs: 1735698997
+  nsecs: 943296670
+frame_id: '' and grasped_object_mask header seq: 189093
+stamp:
+  secs: 1735698997
+  nsecs: 700213194
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698998.197129]: received mask_obs with header seq: 2367
+stamp:
+  secs: 1735698998
+  nsecs: 200725555
+frame_id: '' and grasped_object_mask header seq: 189101
+stamp:
+  secs: 1735698997
+  nsecs: 966657877
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698998.459383]: received mask_obs with header seq: 2368
+stamp:
+  secs: 1735698998
+  nsecs: 462336301
+frame_id: '' and grasped_object_mask header seq: 189109
+stamp:
+  secs: 1735698998
+  nsecs: 233102798
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698998.725313]: received mask_obs with header seq: 2369
+stamp:
+  secs: 1735698998
+  nsecs: 728945255
+frame_id: '' and grasped_object_mask header seq: 189117
+stamp:
+  secs: 1735698998
+  nsecs: 499546289
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698998.986804]: received mask_obs with header seq: 2370
+stamp:
+  secs: 1735698998
+  nsecs: 990421533
+frame_id: '' and grasped_object_mask header seq: 189124
+stamp:
+  secs: 1735698998
+  nsecs: 732683182
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698999.238391]: received mask_obs with header seq: 2371
+stamp:
+  secs: 1735698999
+  nsecs: 242034435
+frame_id: '' and grasped_object_mask header seq: 189132
+stamp:
+  secs: 1735698998
+  nsecs: 999124050
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698999.496799]: received mask_obs with header seq: 2372
+stamp:
+  secs: 1735698999
+  nsecs: 499984979
+frame_id: '' and grasped_object_mask header seq: 189140
+stamp:
+  secs: 1735698999
+  nsecs: 265564442
+frame_id: "camera_color_optical_frame"
+[INFO] [1735698999.759295]: received mask_obs with header seq: 2373
+stamp:
+  secs: 1735698999
+  nsecs: 762934446
+frame_id: '' and grasped_object_mask header seq: 189148
+stamp:
+  secs: 1735698999
+  nsecs: 532007694
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699000.022462]: received mask_obs with header seq: 2374
+stamp:
+  secs: 1735699000
+  nsecs:  25780677
+frame_id: '' and grasped_object_mask header seq: 189156
+stamp:
+  secs: 1735698999
+  nsecs: 798454046
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699000.278992]: received mask_obs with header seq: 2375
+stamp:
+  secs: 1735699000
+  nsecs: 282263278
+frame_id: '' and grasped_object_mask header seq: 189163
+stamp:
+  secs: 1735699000
+  nsecs:  31594992
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699000.533719]: received mask_obs with header seq: 2376
+stamp:
+  secs: 1735699000
+  nsecs: 537147998
+frame_id: '' and grasped_object_mask header seq: 189171
+stamp:
+  secs: 1735699000
+  nsecs: 298040867
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699000.802003]: received mask_obs with header seq: 2377
+stamp:
+  secs: 1735699000
+  nsecs: 805682182
+frame_id: '' and grasped_object_mask header seq: 189179
+stamp:
+  secs: 1735699000
+  nsecs: 564486980
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699001.063765]: received mask_obs with header seq: 2378
+stamp:
+  secs: 1735699001
+  nsecs:  66899061
+frame_id: '' and grasped_object_mask header seq: 189187
+stamp:
+  secs: 1735699000
+  nsecs: 830933809
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699001.324478]: received mask_obs with header seq: 2379
+stamp:
+  secs: 1735699001
+  nsecs: 327923059
+frame_id: '' and grasped_object_mask header seq: 189195
+stamp:
+  secs: 1735699001
+  nsecs:  97380400
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699001.590788]: received mask_obs with header seq: 2380
+stamp:
+  secs: 1735699001
+  nsecs: 594418287
+frame_id: '' and grasped_object_mask header seq: 189203
+stamp:
+  secs: 1735699001
+  nsecs: 363827705
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699001.836803]: received mask_obs with header seq: 2381
+stamp:
+  secs: 1735699001
+  nsecs: 840364694
+frame_id: '' and grasped_object_mask header seq: 189210
+stamp:
+  secs: 1735699001
+  nsecs: 596967936
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699002.107333]: received mask_obs with header seq: 2382
+stamp:
+  secs: 1735699002
+  nsecs: 110989332
+frame_id: '' and grasped_object_mask header seq: 189218
+stamp:
+  secs: 1735699001
+  nsecs: 863415718
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699002.361261]: received mask_obs with header seq: 2383
+stamp:
+  secs: 1735699002
+  nsecs: 364598751
+frame_id: '' and grasped_object_mask header seq: 189226
+stamp:
+  secs: 1735699002
+  nsecs: 129862309
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699002.623075]: received mask_obs with header seq: 2384
+stamp:
+  secs: 1735699002
+  nsecs: 626595258
+frame_id: '' and grasped_object_mask header seq: 189234
+stamp:
+  secs: 1735699002
+  nsecs: 396308899
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699002.886389]: received mask_obs with header seq: 2385
+stamp:
+  secs: 1735699002
+  nsecs: 889822959
+frame_id: '' and grasped_object_mask header seq: 189242
+stamp:
+  secs: 1735699002
+  nsecs: 662753820
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699003.136008]: received mask_obs with header seq: 2386
+stamp:
+  secs: 1735699003
+  nsecs: 139591217
+frame_id: '' and grasped_object_mask header seq: 189249
+stamp:
+  secs: 1735699002
+  nsecs: 895893574
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699003.394297]: received mask_obs with header seq: 2387
+stamp:
+  secs: 1735699003
+  nsecs: 397716999
+frame_id: '' and grasped_object_mask header seq: 189257
+stamp:
+  secs: 1735699003
+  nsecs: 162338495
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699003.655152]: received mask_obs with header seq: 2388
+stamp:
+  secs: 1735699003
+  nsecs: 658610582
+frame_id: '' and grasped_object_mask header seq: 189265
+stamp:
+  secs: 1735699003
+  nsecs: 428782940
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699003.922440]: received mask_obs with header seq: 2389
+stamp:
+  secs: 1735699003
+  nsecs: 926061630
+frame_id: '' and grasped_object_mask header seq: 189273
+stamp:
+  secs: 1735699003
+  nsecs: 695230484
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699004.189992]: received mask_obs with header seq: 2390
+stamp:
+  secs: 1735699004
+  nsecs: 193638563
+frame_id: '' and grasped_object_mask header seq: 189280
+stamp:
+  secs: 1735699003
+  nsecs: 928370714
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699004.454071]: received mask_obs with header seq: 2391
+stamp:
+  secs: 1735699004
+  nsecs: 456613779
+frame_id: '' and grasped_object_mask header seq: 189289
+stamp:
+  secs: 1735699004
+  nsecs: 228123188
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699004.702136]: received mask_obs with header seq: 2392
+stamp:
+  secs: 1735699004
+  nsecs: 705753803
+frame_id: '' and grasped_object_mask header seq: 189296
+stamp:
+  secs: 1735699004
+  nsecs: 461265564
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699004.960566]: received mask_obs with header seq: 2393
+stamp:
+  secs: 1735699004
+  nsecs: 964019060
+frame_id: '' and grasped_object_mask header seq: 189304
+stamp:
+  secs: 1735699004
+  nsecs: 727725506
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699005.222454]: received mask_obs with header seq: 2394
+stamp:
+  secs: 1735699005
+  nsecs: 225285053
+frame_id: '' and grasped_object_mask header seq: 189312
+stamp:
+  secs: 1735699004
+  nsecs: 994185686
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699005.487908]: received mask_obs with header seq: 2395
+stamp:
+  secs: 1735699005
+  nsecs: 490287542
+frame_id: '' and grasped_object_mask header seq: 189320
+stamp:
+  secs: 1735699005
+  nsecs: 260646343
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699005.737480]: received mask_obs with header seq: 2396
+stamp:
+  secs: 1735699005
+  nsecs: 741092920
+frame_id: '' and grasped_object_mask header seq: 189327
+stamp:
+  secs: 1735699005
+  nsecs: 493798256
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699006.002849]: received mask_obs with header seq: 2397
+stamp:
+  secs: 1735699006
+  nsecs:   6274700
+frame_id: '' and grasped_object_mask header seq: 189335
+stamp:
+  secs: 1735699005
+  nsecs: 760252237
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699006.260030]: received mask_obs with header seq: 2398
+stamp:
+  secs: 1735699006
+  nsecs: 263630151
+frame_id: '' and grasped_object_mask header seq: 189343
+stamp:
+  secs: 1735699006
+  nsecs:  26705980
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699006.520135]: received mask_obs with header seq: 2399
+stamp:
+  secs: 1735699006
+  nsecs: 523696899
+frame_id: '' and grasped_object_mask header seq: 189351
+stamp:
+  secs: 1735699006
+  nsecs: 293161392
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699006.786829]: received mask_obs with header seq: 2400
+stamp:
+  secs: 1735699006
+  nsecs: 790471315
+frame_id: '' and grasped_object_mask header seq: 189359
+stamp:
+  secs: 1735699006
+  nsecs: 559619427
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699007.053775]: received mask_obs with header seq: 2401
+stamp:
+  secs: 1735699007
+  nsecs:  56966066
+frame_id: '' and grasped_object_mask header seq: 189366
+stamp:
+  secs: 1735699006
+  nsecs: 792774916
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699007.322485]: received mask_obs with header seq: 2402
+stamp:
+  secs: 1735699007
+  nsecs: 326125621
+frame_id: '' and grasped_object_mask header seq: 189375
+stamp:
+  secs: 1735699007
+  nsecs:  92545748
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699007.588725]: received mask_obs with header seq: 2403
+stamp:
+  secs: 1735699007
+  nsecs: 592277765
+frame_id: '' and grasped_object_mask header seq: 189383
+stamp:
+  secs: 1735699007
+  nsecs: 359010935
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699007.853108]: received mask_obs with header seq: 2404
+stamp:
+  secs: 1735699007
+  nsecs: 856178760
+frame_id: '' and grasped_object_mask header seq: 189391
+stamp:
+  secs: 1735699007
+  nsecs: 625468016
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699008.108865]: received mask_obs with header seq: 2405
+stamp:
+  secs: 1735699008
+  nsecs: 112408399
+frame_id: '' and grasped_object_mask header seq: 189398
+stamp:
+  secs: 1735699007
+  nsecs: 858614683
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699008.364878]: received mask_obs with header seq: 2406
+stamp:
+  secs: 1735699008
+  nsecs: 368359327
+frame_id: '' and grasped_object_mask header seq: 189406
+stamp:
+  secs: 1735699008
+  nsecs: 125070095
+frame_id: "camera_color_optical_frame"
+preempting episode
+[INFO] [1735699008.582319]: Storing episode...
+[INFO] [1735699008.621213]: received mask_obs with header seq: 2407
+stamp:
+  secs: 1735699008
+  nsecs: 624825716
+frame_id: '' and grasped_object_mask header seq: 189414
+stamp:
+  secs: 1735699008
+  nsecs: 391524792
+frame_id: "camera_color_optical_frame"
+[print_text-5] killing on exit
+[rosbag_record-6] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1735699009.391364]: Stored episode 2.
+[INFO] [1735699009.391612]: Saving video...
+[INFO] [1735699009.948517]: Video saved!
+Starting episode 2
+[INFO] [1735699009.973303]: resetting environment
+[INFO] [1735699009.990480]: cleared current plan
+[INFO] [1735699009.990714]: moving to home
+[WARN] [1735699009.826742]: Plan exhausted
+[WARN] [1735699009.864812]: Plan exhausted
+[WARN] [1735699009.905390]: Plan exhausted
+[WARN] [1735699009.945174]: Plan exhausted
+[WARN] [1735699009.984782]: Plan exhausted
+[INFO] [1735699013.295734]: reached home
+[INFO] [1735699013.298904]: reset action history
+[INFO] [1735699016.073723]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735699037.106630]: resetting environment
+[INFO] [1735699037.113392]: cleared current plan
+[INFO] [1735699037.113565]: moving to home
+[INFO] [1735699038.114981]: reached home
+[INFO] [1735699038.115710]: reset action history
+[INFO] [1735699040.823740]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/episode_rosbags/episode_2_2024-12-31-21-37-20.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1748029.log
+started roslaunch server http://158.130.50.37:42527/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [1748824]
+process[print_text-8]: started with pid [1748825]
+process[rosbag_record-9]: started with pid [1748849]
+started bagging!
+For topic gripper_width: timestamp difference is 121743026 for nearest: 1735699041453354659 - target: 1735699041331611633 at idx 12
+[INFO] [1735699041.577489]: received mask_obs with header seq: 2408
+stamp:
+  secs: 1735699041
+  nsecs: 580134868
+frame_id: '' and grasped_object_mask header seq: 190403
+stamp:
+  secs: 1735699041
+  nsecs: 331611633
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699041.808464]: received mask_obs with header seq: 2409
+stamp:
+  secs: 1735699041
+  nsecs: 812038898
+frame_id: '' and grasped_object_mask header seq: 190411
+stamp:
+  secs: 1735699041
+  nsecs: 598056078
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699042.052211]: received mask_obs with header seq: 2410
+stamp:
+  secs: 1735699042
+  nsecs:  55662155
+frame_id: '' and grasped_object_mask header seq: 190418
+stamp:
+  secs: 1735699041
+  nsecs: 831191063
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699042.301993]: received mask_obs with header seq: 2411
+stamp:
+  secs: 1735699042
+  nsecs: 305595159
+frame_id: '' and grasped_object_mask header seq: 190426
+stamp:
+  secs: 1735699042
+  nsecs:  97630024
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699042.540424]: received mask_obs with header seq: 2412
+stamp:
+  secs: 1735699042
+  nsecs: 543929100
+frame_id: '' and grasped_object_mask header seq: 190433
+stamp:
+  secs: 1735699042
+  nsecs: 330764055
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699042.794239]: received mask_obs with header seq: 2413
+stamp:
+  secs: 1735699042
+  nsecs: 797623872
+frame_id: '' and grasped_object_mask header seq: 190440
+stamp:
+  secs: 1735699042
+  nsecs: 563902617
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699043.045931]: received mask_obs with header seq: 2414
+stamp:
+  secs: 1735699043
+  nsecs:  48404932
+frame_id: '' and grasped_object_mask header seq: 190447
+stamp:
+  secs: 1735699042
+  nsecs: 797046900
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699043.306168]: received mask_obs with header seq: 2415
+stamp:
+  secs: 1735699043
+  nsecs: 309446573
+frame_id: '' and grasped_object_mask header seq: 190455
+stamp:
+  secs: 1735699043
+  nsecs:  63497543
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699043.569145]: received mask_obs with header seq: 2416
+stamp:
+  secs: 1735699043
+  nsecs: 572525262
+frame_id: '' and grasped_object_mask header seq: 190463
+stamp:
+  secs: 1735699043
+  nsecs: 329948902
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699043.826202]: received mask_obs with header seq: 2417
+stamp:
+  secs: 1735699043
+  nsecs: 829713582
+frame_id: '' and grasped_object_mask header seq: 190471
+stamp:
+  secs: 1735699043
+  nsecs: 596400499
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699044.091078]: received mask_obs with header seq: 2418
+stamp:
+  secs: 1735699044
+  nsecs:  94595432
+frame_id: '' and grasped_object_mask header seq: 190479
+stamp:
+  secs: 1735699043
+  nsecs: 862853050
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699044.355874]: received mask_obs with header seq: 2419
+stamp:
+  secs: 1735699044
+  nsecs: 358582735
+frame_id: '' and grasped_object_mask header seq: 190487
+stamp:
+  secs: 1735699044
+  nsecs: 129305840
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699044.607204]: received mask_obs with header seq: 2420
+stamp:
+  secs: 1735699044
+  nsecs: 610634565
+frame_id: '' and grasped_object_mask header seq: 190494
+stamp:
+  secs: 1735699044
+  nsecs: 362452745
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699044.866271]: received mask_obs with header seq: 2421
+stamp:
+  secs: 1735699044
+  nsecs: 869912147
+frame_id: '' and grasped_object_mask header seq: 190502
+stamp:
+  secs: 1735699044
+  nsecs: 628900290
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699045.126602]: received mask_obs with header seq: 2422
+stamp:
+  secs: 1735699045
+  nsecs: 130115985
+frame_id: '' and grasped_object_mask header seq: 190510
+stamp:
+  secs: 1735699044
+  nsecs: 895344973
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699045.395059]: received mask_obs with header seq: 2423
+stamp:
+  secs: 1735699045
+  nsecs: 398665666
+frame_id: '' and grasped_object_mask header seq: 190518
+stamp:
+  secs: 1735699045
+  nsecs: 161790133
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699045.654867]: received mask_obs with header seq: 2424
+stamp:
+  secs: 1735699045
+  nsecs: 658479690
+frame_id: '' and grasped_object_mask header seq: 190526
+stamp:
+  secs: 1735699045
+  nsecs: 428233862
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699045.922767]: received mask_obs with header seq: 2425
+stamp:
+  secs: 1735699045
+  nsecs: 926174163
+frame_id: '' and grasped_object_mask header seq: 190534
+stamp:
+  secs: 1735699045
+  nsecs: 696651936
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699046.185037]: received mask_obs with header seq: 2426
+stamp:
+  secs: 1735699046
+  nsecs: 187717914
+frame_id: '' and grasped_object_mask header seq: 190541
+stamp:
+  secs: 1735699045
+  nsecs: 932575226
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699046.434570]: received mask_obs with header seq: 2427
+stamp:
+  secs: 1735699046
+  nsecs: 437827348
+frame_id: '' and grasped_object_mask header seq: 190549
+stamp:
+  secs: 1735699046
+  nsecs: 202345848
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699046.694419]: received mask_obs with header seq: 2428
+stamp:
+  secs: 1735699046
+  nsecs: 697840690
+frame_id: '' and grasped_object_mask header seq: 190557
+stamp:
+  secs: 1735699046
+  nsecs: 472271681
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699046.952104]: received mask_obs with header seq: 2429
+stamp:
+  secs: 1735699046
+  nsecs: 955549478
+frame_id: '' and grasped_object_mask header seq: 190565
+stamp:
+  secs: 1735699046
+  nsecs: 739457607
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699047.214728]: received mask_obs with header seq: 2430
+stamp:
+  secs: 1735699047
+  nsecs: 218352794
+frame_id: '' and grasped_object_mask header seq: 190573
+stamp:
+  secs: 1735699047
+  nsecs:   5584717
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699047.477575]: received mask_obs with header seq: 2431
+stamp:
+  secs: 1735699047
+  nsecs: 477717399
+frame_id: '' and grasped_object_mask header seq: 190580
+stamp:
+  secs: 1735699047
+  nsecs: 238427162
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699047.727684]: received mask_obs with header seq: 2432
+stamp:
+  secs: 1735699047
+  nsecs: 731240749
+frame_id: '' and grasped_object_mask header seq: 190588
+stamp:
+  secs: 1735699047
+  nsecs: 504514456
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699047.986185]: received mask_obs with header seq: 2433
+stamp:
+  secs: 1735699047
+  nsecs: 989764451
+frame_id: '' and grasped_object_mask header seq: 190596
+stamp:
+  secs: 1735699047
+  nsecs: 770633221
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699048.250860]: received mask_obs with header seq: 2434
+stamp:
+  secs: 1735699048
+  nsecs: 254379034
+frame_id: '' and grasped_object_mask header seq: 190604
+stamp:
+  secs: 1735699048
+  nsecs:  36749125
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699048.512792]: received mask_obs with header seq: 2435
+stamp:
+  secs: 1735699048
+  nsecs: 516253709
+frame_id: '' and grasped_object_mask header seq: 190611
+stamp:
+  secs: 1735699048
+  nsecs: 269582987
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699048.763305]: received mask_obs with header seq: 2436
+stamp:
+  secs: 1735699048
+  nsecs: 766881942
+frame_id: '' and grasped_object_mask header seq: 190619
+stamp:
+  secs: 1735699048
+  nsecs: 535658360
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699049.019939]: received mask_obs with header seq: 2437
+stamp:
+  secs: 1735699049
+  nsecs:  23441314
+frame_id: '' and grasped_object_mask header seq: 190627
+stamp:
+  secs: 1735699048
+  nsecs: 801792383
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699049.283377]: received mask_obs with header seq: 2438
+stamp:
+  secs: 1735699049
+  nsecs: 286700248
+frame_id: '' and grasped_object_mask header seq: 190635
+stamp:
+  secs: 1735699049
+  nsecs:  67917109
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699049.548545]: received mask_obs with header seq: 2439
+stamp:
+  secs: 1735699049
+  nsecs: 551879882
+frame_id: '' and grasped_object_mask header seq: 190643
+stamp:
+  secs: 1735699049
+  nsecs: 334022045
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699049.795772]: received mask_obs with header seq: 2440
+stamp:
+  secs: 1735699049
+  nsecs: 799351453
+frame_id: '' and grasped_object_mask header seq: 190650
+stamp:
+  secs: 1735699049
+  nsecs: 566845179
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699050.054509]: received mask_obs with header seq: 2441
+stamp:
+  secs: 1735699050
+  nsecs:  58075189
+frame_id: '' and grasped_object_mask header seq: 190658
+stamp:
+  secs: 1735699049
+  nsecs: 832991123
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699050.314872]: received mask_obs with header seq: 2442
+stamp:
+  secs: 1735699050
+  nsecs: 318149805
+frame_id: '' and grasped_object_mask header seq: 190666
+stamp:
+  secs: 1735699050
+  nsecs:  99117517
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699050.583410]: received mask_obs with header seq: 2443
+stamp:
+  secs: 1735699050
+  nsecs: 586957216
+frame_id: '' and grasped_object_mask header seq: 190674
+stamp:
+  secs: 1735699050
+  nsecs: 365221977
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699050.848639]: received mask_obs with header seq: 2444
+stamp:
+  secs: 1735699050
+  nsecs: 851912975
+frame_id: '' and grasped_object_mask header seq: 190681
+stamp:
+  secs: 1735699050
+  nsecs: 598058224
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699051.117107]: received mask_obs with header seq: 2445
+stamp:
+  secs: 1735699051
+  nsecs: 120178461
+frame_id: '' and grasped_object_mask header seq: 190690
+stamp:
+  secs: 1735699050
+  nsecs: 897480011
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699051.378992]: received mask_obs with header seq: 2446
+stamp:
+  secs: 1735699051
+  nsecs: 382216691
+frame_id: '' and grasped_object_mask header seq: 190698
+stamp:
+  secs: 1735699051
+  nsecs: 163613081
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699051.630338]: received mask_obs with header seq: 2447
+stamp:
+  secs: 1735699051
+  nsecs: 633711814
+frame_id: '' and grasped_object_mask header seq: 190705
+stamp:
+  secs: 1735699051
+  nsecs: 396461487
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699051.885324]: received mask_obs with header seq: 2448
+stamp:
+  secs: 1735699051
+  nsecs: 888708353
+frame_id: '' and grasped_object_mask header seq: 190713
+stamp:
+  secs: 1735699051
+  nsecs: 662581682
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699052.147375]: received mask_obs with header seq: 2449
+stamp:
+  secs: 1735699052
+  nsecs: 151031970
+frame_id: '' and grasped_object_mask header seq: 190721
+stamp:
+  secs: 1735699051
+  nsecs: 928725719
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699052.401514]: received mask_obs with header seq: 2450
+stamp:
+  secs: 1735699052
+  nsecs: 404921054
+frame_id: '' and grasped_object_mask header seq: 190728
+stamp:
+  secs: 1735699052
+  nsecs: 161583900
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699052.655087]: received mask_obs with header seq: 2451
+stamp:
+  secs: 1735699052
+  nsecs: 658672809
+frame_id: '' and grasped_object_mask header seq: 190736
+stamp:
+  secs: 1735699052
+  nsecs: 427689791
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699052.917098]: received mask_obs with header seq: 2452
+stamp:
+  secs: 1735699052
+  nsecs: 920570135
+frame_id: '' and grasped_object_mask header seq: 190744
+stamp:
+  secs: 1735699052
+  nsecs: 693813324
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699053.180305]: received mask_obs with header seq: 2453
+stamp:
+  secs: 1735699053
+  nsecs: 183734178
+frame_id: '' and grasped_object_mask header seq: 190752
+stamp:
+  secs: 1735699052
+  nsecs: 959955215
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699053.443105]: received mask_obs with header seq: 2454
+stamp:
+  secs: 1735699053
+  nsecs: 446181058
+frame_id: '' and grasped_object_mask header seq: 190759
+stamp:
+  secs: 1735699053
+  nsecs: 192813396
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699053.695120]: received mask_obs with header seq: 2455
+stamp:
+  secs: 1735699053
+  nsecs: 698638916
+frame_id: '' and grasped_object_mask header seq: 190767
+stamp:
+  secs: 1735699053
+  nsecs: 458917141
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699053.952657]: received mask_obs with header seq: 2456
+stamp:
+  secs: 1735699053
+  nsecs: 956129312
+frame_id: '' and grasped_object_mask header seq: 190775
+stamp:
+  secs: 1735699053
+  nsecs: 725042582
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699054.212914]: received mask_obs with header seq: 2457
+stamp:
+  secs: 1735699054
+  nsecs: 216531038
+frame_id: '' and grasped_object_mask header seq: 190783
+stamp:
+  secs: 1735699053
+  nsecs: 991171837
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699054.480061]: received mask_obs with header seq: 2458
+stamp:
+  secs: 1735699054
+  nsecs: 483448028
+frame_id: '' and grasped_object_mask header seq: 190791
+stamp:
+  secs: 1735699054
+  nsecs: 257281303
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699054.741922]: received mask_obs with header seq: 2459
+stamp:
+  secs: 1735699054
+  nsecs: 745362281
+frame_id: '' and grasped_object_mask header seq: 190799
+stamp:
+  secs: 1735699054
+  nsecs: 523368359
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699054.999864]: received mask_obs with header seq: 2460
+stamp:
+  secs: 1735699055
+  nsecs:   3474950
+frame_id: '' and grasped_object_mask header seq: 190806
+stamp:
+  secs: 1735699054
+  nsecs: 756240129
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699055.251796]: received mask_obs with header seq: 2461
+stamp:
+  secs: 1735699055
+  nsecs: 255349397
+frame_id: '' and grasped_object_mask header seq: 190814
+stamp:
+  secs: 1735699055
+  nsecs:  22373676
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699055.514308]: received mask_obs with header seq: 2462
+stamp:
+  secs: 1735699055
+  nsecs: 517908811
+frame_id: '' and grasped_object_mask header seq: 190822
+stamp:
+  secs: 1735699055
+  nsecs: 288485765
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699055.776129]: received mask_obs with header seq: 2463
+stamp:
+  secs: 1735699055
+  nsecs: 779721736
+frame_id: '' and grasped_object_mask header seq: 190830
+stamp:
+  secs: 1735699055
+  nsecs: 554577112
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699056.035161]: received mask_obs with header seq: 2464
+stamp:
+  secs: 1735699056
+  nsecs:  37570476
+frame_id: '' and grasped_object_mask header seq: 190838
+stamp:
+  secs: 1735699055
+  nsecs: 820733786
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699056.292638]: received mask_obs with header seq: 2465
+stamp:
+  secs: 1735699056
+  nsecs: 295903921
+frame_id: '' and grasped_object_mask header seq: 190845
+stamp:
+  secs: 1735699056
+  nsecs:  53604841
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699056.550846]: received mask_obs with header seq: 2466
+stamp:
+  secs: 1735699056
+  nsecs: 554469823
+frame_id: '' and grasped_object_mask header seq: 190853
+stamp:
+  secs: 1735699056
+  nsecs: 319723606
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699056.810668]: received mask_obs with header seq: 2467
+stamp:
+  secs: 1735699056
+  nsecs: 813902378
+frame_id: '' and grasped_object_mask header seq: 190861
+stamp:
+  secs: 1735699056
+  nsecs: 585830927
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699057.075986]: received mask_obs with header seq: 2468
+stamp:
+  secs: 1735699057
+  nsecs:  79620838
+frame_id: '' and grasped_object_mask header seq: 190869
+stamp:
+  secs: 1735699056
+  nsecs: 851978064
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699057.329163]: received mask_obs with header seq: 2469
+stamp:
+  secs: 1735699057
+  nsecs: 332728385
+frame_id: '' and grasped_object_mask header seq: 190876
+stamp:
+  secs: 1735699057
+  nsecs:  84838629
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699057.587849]: received mask_obs with header seq: 2470
+stamp:
+  secs: 1735699057
+  nsecs: 591263771
+frame_id: '' and grasped_object_mask header seq: 190884
+stamp:
+  secs: 1735699057
+  nsecs: 350947380
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699057.856030]: received mask_obs with header seq: 2471
+stamp:
+  secs: 1735699057
+  nsecs: 859609842
+frame_id: '' and grasped_object_mask header seq: 190892
+stamp:
+  secs: 1735699057
+  nsecs: 617054462
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699058.121189]: received mask_obs with header seq: 2472
+stamp:
+  secs: 1735699058
+  nsecs: 124782800
+frame_id: '' and grasped_object_mask header seq: 190900
+stamp:
+  secs: 1735699057
+  nsecs: 883201122
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699058.375368]: received mask_obs with header seq: 2473
+stamp:
+  secs: 1735699058
+  nsecs: 378292798
+frame_id: '' and grasped_object_mask header seq: 190908
+stamp:
+  secs: 1735699058
+  nsecs: 149327040
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699058.643595]: received mask_obs with header seq: 2474
+stamp:
+  secs: 1735699058
+  nsecs: 646358013
+frame_id: '' and grasped_object_mask header seq: 190916
+stamp:
+  secs: 1735699058
+  nsecs: 415432215
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699058.894283]: received mask_obs with header seq: 2475
+stamp:
+  secs: 1735699058
+  nsecs: 897898435
+frame_id: '' and grasped_object_mask header seq: 190923
+stamp:
+  secs: 1735699058
+  nsecs: 648287058
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699059.155407]: received mask_obs with header seq: 2476
+stamp:
+  secs: 1735699059
+  nsecs: 159036874
+frame_id: '' and grasped_object_mask header seq: 190931
+stamp:
+  secs: 1735699058
+  nsecs: 914432049
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699059.412640]: received mask_obs with header seq: 2477
+stamp:
+  secs: 1735699059
+  nsecs: 416250228
+frame_id: '' and grasped_object_mask header seq: 190939
+stamp:
+  secs: 1735699059
+  nsecs: 180557966
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699059.673812]: received mask_obs with header seq: 2478
+stamp:
+  secs: 1735699059
+  nsecs: 677457332
+frame_id: '' and grasped_object_mask header seq: 190947
+stamp:
+  secs: 1735699059
+  nsecs: 446661949
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699059.945354]: received mask_obs with header seq: 2479
+stamp:
+  secs: 1735699059
+  nsecs: 948773145
+frame_id: '' and grasped_object_mask header seq: 190955
+stamp:
+  secs: 1735699059
+  nsecs: 712793827
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699060.206644]: received mask_obs with header seq: 2480
+stamp:
+  secs: 1735699060
+  nsecs: 210250139
+frame_id: '' and grasped_object_mask header seq: 190963
+stamp:
+  secs: 1735699059
+  nsecs: 978934288
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699060.469416]: received mask_obs with header seq: 2481
+stamp:
+  secs: 1735699060
+  nsecs: 472780942
+frame_id: '' and grasped_object_mask header seq: 190971
+stamp:
+  secs: 1735699060
+  nsecs: 245054007
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699060.739316]: received mask_obs with header seq: 2482
+stamp:
+  secs: 1735699060
+  nsecs: 742978096
+frame_id: '' and grasped_object_mask header seq: 190978
+stamp:
+  secs: 1735699060
+  nsecs: 477892399
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699061.020123]: received mask_obs with header seq: 2483
+stamp:
+  secs: 1735699061
+  nsecs:  23756027
+frame_id: '' and grasped_object_mask header seq: 190987
+stamp:
+  secs: 1735699060
+  nsecs: 777287245
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699061.272262]: received mask_obs with header seq: 2484
+stamp:
+  secs: 1735699061
+  nsecs: 275890350
+frame_id: '' and grasped_object_mask header seq: 190995
+stamp:
+  secs: 1735699061
+  nsecs:  43407202
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699061.538643]: received mask_obs with header seq: 2485
+stamp:
+  secs: 1735699061
+  nsecs: 542310476
+frame_id: '' and grasped_object_mask header seq: 191002
+stamp:
+  secs: 1735699061
+  nsecs: 276244640
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699061.788825]: received mask_obs with header seq: 2486
+stamp:
+  secs: 1735699061
+  nsecs: 792433500
+frame_id: '' and grasped_object_mask header seq: 191010
+stamp:
+  secs: 1735699061
+  nsecs: 542323589
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699062.045917]: received mask_obs with header seq: 2487
+stamp:
+  secs: 1735699062
+  nsecs:  49360752
+frame_id: '' and grasped_object_mask header seq: 191018
+stamp:
+  secs: 1735699061
+  nsecs: 809964657
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699062.309091]: received mask_obs with header seq: 2488
+stamp:
+  secs: 1735699062
+  nsecs: 312565326
+frame_id: '' and grasped_object_mask header seq: 191026
+stamp:
+  secs: 1735699062
+  nsecs:  77990770
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699062.572717]: received mask_obs with header seq: 2489
+stamp:
+  secs: 1735699062
+  nsecs: 576189517
+frame_id: '' and grasped_object_mask header seq: 191034
+stamp:
+  secs: 1735699062
+  nsecs: 346173525
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699062.820754]: received mask_obs with header seq: 2490
+stamp:
+  secs: 1735699062
+  nsecs: 824119329
+frame_id: '' and grasped_object_mask header seq: 191041
+stamp:
+  secs: 1735699062
+  nsecs: 580990314
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699063.078293]: received mask_obs with header seq: 2491
+stamp:
+  secs: 1735699063
+  nsecs:  81950664
+frame_id: '' and grasped_object_mask header seq: 191049
+stamp:
+  secs: 1735699062
+  nsecs: 847445488
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699063.339674]: received mask_obs with header seq: 2492
+stamp:
+  secs: 1735699063
+  nsecs: 343206405
+frame_id: '' and grasped_object_mask header seq: 191057
+stamp:
+  secs: 1735699063
+  nsecs: 113902807
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699063.611139]: received mask_obs with header seq: 2493
+stamp:
+  secs: 1735699063
+  nsecs: 614727258
+frame_id: '' and grasped_object_mask header seq: 191065
+stamp:
+  secs: 1735699063
+  nsecs: 380359650
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699063.870343]: received mask_obs with header seq: 2494
+stamp:
+  secs: 1735699063
+  nsecs: 873820304
+frame_id: '' and grasped_object_mask header seq: 191073
+stamp:
+  secs: 1735699063
+  nsecs: 646816492
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699064.135623]: received mask_obs with header seq: 2495
+stamp:
+  secs: 1735699064
+  nsecs: 139267444
+frame_id: '' and grasped_object_mask header seq: 191081
+stamp:
+  secs: 1735699063
+  nsecs: 913271666
+frame_id: "camera_color_optical_frame"
+preempting episode
+[INFO] [1735699064.373305]: Storing episode...
+[INFO] [1735699064.385622]: received mask_obs with header seq: 2496
+stamp:
+  secs: 1735699064
+  nsecs: 388615846
+frame_id: '' and grasped_object_mask header seq: 191088
+stamp:
+  secs: 1735699064
+  nsecs: 146420717
+frame_id: "camera_color_optical_frame"
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1735699065.168162]: Stored episode 3.
+[INFO] [1735699065.169535]: Saving video...
+[INFO] [1735699065.852432]: Video saved!
+Starting episode 3
+[INFO] [1735699065.917550]: resetting environment
+[INFO] [1735699065.942027]: cleared current plan
+[INFO] [1735699065.944577]: moving to home
+[WARN] [1735699065.584505]: Plan exhausted
+[WARN] [1735699065.624469]: Plan exhausted
+[WARN] [1735699065.666774]: Plan exhausted
+[WARN] [1735699065.705458]: Plan exhausted
+[WARN] [1735699065.749488]: Plan exhausted
+[WARN] [1735699065.791628]: Plan exhausted
+[WARN] [1735699065.829733]: Plan exhausted
+[WARN] [1735699065.864857]: Plan exhausted
+[WARN] [1735699065.904913]: Plan exhausted
+[INFO] [1735699069.148280]: reached home
+[INFO] [1735699069.153299]: reset action history
+[INFO] [1735699072.098930]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735699083.915224]: resetting environment
+[INFO] [1735699083.930502]: cleared current plan
+[INFO] [1735699083.930707]: moving to home
+[INFO] [1735699084.932572]: reached home
+[INFO] [1735699084.934790]: reset action history
+[INFO] [1735699087.671191]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/episode_rosbags/episode_3_2024-12-31-21-38-07.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1748029.log
+started roslaunch server http://158.130.50.37:36899/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [1748970]
+process[print_text-11]: started with pid [1748971]
+process[rosbag_record-12]: started with pid [1748972]
+started bagging!
+[INFO] [1735699088.361941]: received mask_obs with header seq: 2497
+stamp:
+  secs: 1735699088
+  nsecs: 364953994
+frame_id: '' and grasped_object_mask header seq: 191809
+stamp:
+  secs: 1735699088
+  nsecs: 160373688
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699088.599897]: received mask_obs with header seq: 2498
+stamp:
+  secs: 1735699088
+  nsecs: 603453159
+frame_id: '' and grasped_object_mask header seq: 191816
+stamp:
+  secs: 1735699088
+  nsecs: 393517733
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699088.830487]: received mask_obs with header seq: 2499
+stamp:
+  secs: 1735699088
+  nsecs: 833953380
+frame_id: '' and grasped_object_mask header seq: 191823
+stamp:
+  secs: 1735699088
+  nsecs: 626660347
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699089.076870]: received mask_obs with header seq: 2500
+stamp:
+  secs: 1735699089
+  nsecs:  78553438
+frame_id: '' and grasped_object_mask header seq: 191830
+stamp:
+  secs: 1735699088
+  nsecs: 859801054
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699089.335712]: received mask_obs with header seq: 2501
+stamp:
+  secs: 1735699089
+  nsecs: 339224100
+frame_id: '' and grasped_object_mask header seq: 191837
+stamp:
+  secs: 1735699089
+  nsecs:  92941523
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699089.577852]: received mask_obs with header seq: 2502
+stamp:
+  secs: 1735699089
+  nsecs: 581456184
+frame_id: '' and grasped_object_mask header seq: 191845
+stamp:
+  secs: 1735699089
+  nsecs: 359386206
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699089.844318]: received mask_obs with header seq: 2503
+stamp:
+  secs: 1735699089
+  nsecs: 847900390
+frame_id: '' and grasped_object_mask header seq: 191853
+stamp:
+  secs: 1735699089
+  nsecs: 625831604
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699090.109037]: received mask_obs with header seq: 2504
+stamp:
+  secs: 1735699090
+  nsecs: 111176252
+frame_id: '' and grasped_object_mask header seq: 191860
+stamp:
+  secs: 1735699089
+  nsecs: 861439705
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699090.364202]: received mask_obs with header seq: 2505
+stamp:
+  secs: 1735699090
+  nsecs: 367674589
+frame_id: '' and grasped_object_mask header seq: 191868
+stamp:
+  secs: 1735699090
+  nsecs: 131086349
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699090.619822]: received mask_obs with header seq: 2506
+stamp:
+  secs: 1735699090
+  nsecs: 623437166
+frame_id: '' and grasped_object_mask header seq: 191876
+stamp:
+  secs: 1735699090
+  nsecs: 400887489
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699090.883690]: received mask_obs with header seq: 2507
+stamp:
+  secs: 1735699090
+  nsecs: 887162446
+frame_id: '' and grasped_object_mask header seq: 191884
+stamp:
+  secs: 1735699090
+  nsecs: 670635939
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699091.153147]: received mask_obs with header seq: 2508
+stamp:
+  secs: 1735699091
+  nsecs: 156690359
+frame_id: '' and grasped_object_mask header seq: 191892
+stamp:
+  secs: 1735699090
+  nsecs: 936800480
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699091.422475]: received mask_obs with header seq: 2509
+stamp:
+  secs: 1735699091
+  nsecs: 426134586
+frame_id: '' and grasped_object_mask header seq: 191900
+stamp:
+  secs: 1735699091
+  nsecs: 202944994
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699091.684151]: received mask_obs with header seq: 2510
+stamp:
+  secs: 1735699091
+  nsecs: 687690734
+frame_id: '' and grasped_object_mask header seq: 191908
+stamp:
+  secs: 1735699091
+  nsecs: 469069481
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699091.949670]: received mask_obs with header seq: 2511
+stamp:
+  secs: 1735699091
+  nsecs: 952944993
+frame_id: '' and grasped_object_mask header seq: 191916
+stamp:
+  secs: 1735699091
+  nsecs: 735189915
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699092.208474]: received mask_obs with header seq: 2512
+stamp:
+  secs: 1735699092
+  nsecs: 207942724
+frame_id: '' and grasped_object_mask header seq: 191923
+stamp:
+  secs: 1735699091
+  nsecs: 968060017
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699092.459028]: received mask_obs with header seq: 2513
+stamp:
+  secs: 1735699092
+  nsecs: 462634325
+frame_id: '' and grasped_object_mask header seq: 191931
+stamp:
+  secs: 1735699092
+  nsecs: 234177113
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699092.719554]: received mask_obs with header seq: 2514
+stamp:
+  secs: 1735699092
+  nsecs: 723052501
+frame_id: '' and grasped_object_mask header seq: 191939
+stamp:
+  secs: 1735699092
+  nsecs: 500273705
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699092.984615]: received mask_obs with header seq: 2515
+stamp:
+  secs: 1735699092
+  nsecs: 987518548
+frame_id: '' and grasped_object_mask header seq: 191947
+stamp:
+  secs: 1735699092
+  nsecs: 766383648
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699093.253725]: received mask_obs with header seq: 2516
+stamp:
+  secs: 1735699093
+  nsecs: 257408618
+frame_id: '' and grasped_object_mask header seq: 191955
+stamp:
+  secs: 1735699093
+  nsecs:  32520533
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699093.517551]: received mask_obs with header seq: 2517
+stamp:
+  secs: 1735699093
+  nsecs: 520888805
+frame_id: '' and grasped_object_mask header seq: 191963
+stamp:
+  secs: 1735699093
+  nsecs: 298636198
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699093.781681]: received mask_obs with header seq: 2518
+stamp:
+  secs: 1735699093
+  nsecs: 785312652
+frame_id: '' and grasped_object_mask header seq: 191971
+stamp:
+  secs: 1735699093
+  nsecs: 564730883
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699094.053541]: received mask_obs with header seq: 2519
+stamp:
+  secs: 1735699094
+  nsecs:  57055711
+frame_id: '' and grasped_object_mask header seq: 191979
+stamp:
+  secs: 1735699093
+  nsecs: 830862999
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699094.317225]: received mask_obs with header seq: 2520
+stamp:
+  secs: 1735699094
+  nsecs: 320324659
+frame_id: '' and grasped_object_mask header seq: 191987
+stamp:
+  secs: 1735699094
+  nsecs:  97001553
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699094.584248]: received mask_obs with header seq: 2521
+stamp:
+  secs: 1735699094
+  nsecs: 587893486
+frame_id: '' and grasped_object_mask header seq: 191995
+stamp:
+  secs: 1735699094
+  nsecs: 363119364
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699094.838167]: received mask_obs with header seq: 2522
+stamp:
+  secs: 1735699094
+  nsecs: 839799404
+frame_id: '' and grasped_object_mask header seq: 192002
+stamp:
+  secs: 1735699094
+  nsecs: 595956087
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699095.094195]: received mask_obs with header seq: 2523
+stamp:
+  secs: 1735699095
+  nsecs:  97459793
+frame_id: '' and grasped_object_mask header seq: 192010
+stamp:
+  secs: 1735699094
+  nsecs: 862095594
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699095.347836]: received mask_obs with header seq: 2524
+stamp:
+  secs: 1735699095
+  nsecs: 351509332
+frame_id: '' and grasped_object_mask header seq: 192018
+stamp:
+  secs: 1735699095
+  nsecs: 128229618
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699095.614822]: received mask_obs with header seq: 2525
+stamp:
+  secs: 1735699095
+  nsecs: 618428468
+frame_id: '' and grasped_object_mask header seq: 192026
+stamp:
+  secs: 1735699095
+  nsecs: 394343376
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699095.870067]: received mask_obs with header seq: 2526
+stamp:
+  secs: 1735699095
+  nsecs: 872700929
+frame_id: '' and grasped_object_mask header seq: 192033
+stamp:
+  secs: 1735699095
+  nsecs: 627176285
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699096.129437]: received mask_obs with header seq: 2527
+stamp:
+  secs: 1735699096
+  nsecs: 132864952
+frame_id: '' and grasped_object_mask header seq: 192041
+stamp:
+  secs: 1735699095
+  nsecs: 893323660
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699096.389628]: received mask_obs with header seq: 2528
+stamp:
+  secs: 1735699096
+  nsecs: 393060922
+frame_id: '' and grasped_object_mask header seq: 192049
+stamp:
+  secs: 1735699096
+  nsecs: 159457207
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699096.641831]: received mask_obs with header seq: 2529
+stamp:
+  secs: 1735699096
+  nsecs: 645237684
+frame_id: '' and grasped_object_mask header seq: 192056
+stamp:
+  secs: 1735699096
+  nsecs: 392307997
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699096.891611]: received mask_obs with header seq: 2530
+stamp:
+  secs: 1735699096
+  nsecs: 895194292
+frame_id: '' and grasped_object_mask header seq: 192064
+stamp:
+  secs: 1735699096
+  nsecs: 658414364
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699097.150820]: received mask_obs with header seq: 2531
+stamp:
+  secs: 1735699097
+  nsecs: 154433012
+frame_id: '' and grasped_object_mask header seq: 192072
+stamp:
+  secs: 1735699096
+  nsecs: 924568653
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699097.411498]: received mask_obs with header seq: 2532
+stamp:
+  secs: 1735699097
+  nsecs: 414494514
+frame_id: '' and grasped_object_mask header seq: 192080
+stamp:
+  secs: 1735699097
+  nsecs: 190701485
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699097.674763]: received mask_obs with header seq: 2533
+stamp:
+  secs: 1735699097
+  nsecs: 678202152
+frame_id: '' and grasped_object_mask header seq: 192088
+stamp:
+  secs: 1735699097
+  nsecs: 456813574
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699097.929926]: received mask_obs with header seq: 2534
+stamp:
+  secs: 1735699097
+  nsecs: 933571338
+frame_id: '' and grasped_object_mask header seq: 192095
+stamp:
+  secs: 1735699097
+  nsecs: 689655781
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699098.188628]: received mask_obs with header seq: 2535
+stamp:
+  secs: 1735699098
+  nsecs: 192103147
+frame_id: '' and grasped_object_mask header seq: 192103
+stamp:
+  secs: 1735699097
+  nsecs: 955813169
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699098.443364]: received mask_obs with header seq: 2536
+stamp:
+  secs: 1735699098
+  nsecs: 446956872
+frame_id: '' and grasped_object_mask header seq: 192111
+stamp:
+  secs: 1735699098
+  nsecs: 221951008
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699098.710802]: received mask_obs with header seq: 2537
+stamp:
+  secs: 1735699098
+  nsecs: 714448451
+frame_id: '' and grasped_object_mask header seq: 192119
+stamp:
+  secs: 1735699098
+  nsecs: 488068104
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699098.973068]: received mask_obs with header seq: 2538
+stamp:
+  secs: 1735699098
+  nsecs: 975677251
+frame_id: '' and grasped_object_mask header seq: 192126
+stamp:
+  secs: 1735699098
+  nsecs: 720923185
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699099.244058]: received mask_obs with header seq: 2539
+stamp:
+  secs: 1735699099
+  nsecs: 247611284
+frame_id: '' and grasped_object_mask header seq: 192134
+stamp:
+  secs: 1735699098
+  nsecs: 987074614
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699099.506773]: received mask_obs with header seq: 2540
+stamp:
+  secs: 1735699099
+  nsecs: 509984970
+frame_id: '' and grasped_object_mask header seq: 192143
+stamp:
+  secs: 1735699099
+  nsecs: 286471605
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699099.759308]: received mask_obs with header seq: 2541
+stamp:
+  secs: 1735699099
+  nsecs: 762623786
+frame_id: '' and grasped_object_mask header seq: 192150
+stamp:
+  secs: 1735699099
+  nsecs: 519318104
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699100.021953]: received mask_obs with header seq: 2542
+stamp:
+  secs: 1735699100
+  nsecs:  25402307
+frame_id: '' and grasped_object_mask header seq: 192158
+stamp:
+  secs: 1735699099
+  nsecs: 785444975
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699100.278639]: received mask_obs with header seq: 2543
+stamp:
+  secs: 1735699100
+  nsecs: 281711339
+frame_id: '' and grasped_object_mask header seq: 192166
+stamp:
+  secs: 1735699100
+  nsecs:  51587582
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699100.546273]: received mask_obs with header seq: 2544
+stamp:
+  secs: 1735699100
+  nsecs: 548868417
+frame_id: '' and grasped_object_mask header seq: 192174
+stamp:
+  secs: 1735699100
+  nsecs: 317709446
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699100.805447]: received mask_obs with header seq: 2545
+stamp:
+  secs: 1735699100
+  nsecs: 808687210
+frame_id: '' and grasped_object_mask header seq: 192181
+stamp:
+  secs: 1735699100
+  nsecs: 550549507
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699101.076314]: received mask_obs with header seq: 2546
+stamp:
+  secs: 1735699101
+  nsecs:  79546213
+frame_id: '' and grasped_object_mask header seq: 192189
+stamp:
+  secs: 1735699100
+  nsecs: 816682100
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699101.340690]: received mask_obs with header seq: 2547
+stamp:
+  secs: 1735699101
+  nsecs: 344182729
+frame_id: '' and grasped_object_mask header seq: 192198
+stamp:
+  secs: 1735699101
+  nsecs: 116090536
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699101.608769]: received mask_obs with header seq: 2548
+stamp:
+  secs: 1735699101
+  nsecs: 612317085
+frame_id: '' and grasped_object_mask header seq: 192206
+stamp:
+  secs: 1735699101
+  nsecs: 382209539
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699101.873132]: received mask_obs with header seq: 2549
+stamp:
+  secs: 1735699101
+  nsecs: 876718044
+frame_id: '' and grasped_object_mask header seq: 192213
+stamp:
+  secs: 1735699101
+  nsecs: 615047216
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699102.135821]: received mask_obs with header seq: 2550
+stamp:
+  secs: 1735699102
+  nsecs: 139369487
+frame_id: '' and grasped_object_mask header seq: 192221
+stamp:
+  secs: 1735699101
+  nsecs: 881184340
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699102.392286]: received mask_obs with header seq: 2551
+stamp:
+  secs: 1735699102
+  nsecs: 395712852
+frame_id: '' and grasped_object_mask header seq: 192229
+stamp:
+  secs: 1735699102
+  nsecs: 147308111
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699102.652271]: received mask_obs with header seq: 2552
+stamp:
+  secs: 1735699102
+  nsecs: 655862569
+frame_id: '' and grasped_object_mask header seq: 192237
+stamp:
+  secs: 1735699102
+  nsecs: 413411856
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699102.914222]: received mask_obs with header seq: 2553
+stamp:
+  secs: 1735699102
+  nsecs: 917770862
+frame_id: '' and grasped_object_mask header seq: 192245
+stamp:
+  secs: 1735699102
+  nsecs: 679504871
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699103.174761]: received mask_obs with header seq: 2554
+stamp:
+  secs: 1735699103
+  nsecs: 178208589
+frame_id: '' and grasped_object_mask header seq: 192253
+stamp:
+  secs: 1735699102
+  nsecs: 945659161
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699103.439931]: received mask_obs with header seq: 2555
+stamp:
+  secs: 1735699103
+  nsecs: 443554639
+frame_id: '' and grasped_object_mask header seq: 192261
+stamp:
+  secs: 1735699103
+  nsecs: 211793423
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699103.704759]: received mask_obs with header seq: 2556
+stamp:
+  secs: 1735699103
+  nsecs: 708259105
+frame_id: '' and grasped_object_mask header seq: 192269
+stamp:
+  secs: 1735699103
+  nsecs: 477907181
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699103.968814]: received mask_obs with header seq: 2557
+stamp:
+  secs: 1735699103
+  nsecs: 971932888
+frame_id: '' and grasped_object_mask header seq: 192276
+stamp:
+  secs: 1735699103
+  nsecs: 710757971
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699104.222598]: received mask_obs with header seq: 2558
+stamp:
+  secs: 1735699104
+  nsecs: 226249694
+frame_id: '' and grasped_object_mask header seq: 192284
+stamp:
+  secs: 1735699103
+  nsecs: 976903439
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699104.484916]: received mask_obs with header seq: 2559
+stamp:
+  secs: 1735699104
+  nsecs: 488584995
+frame_id: '' and grasped_object_mask header seq: 192292
+stamp:
+  secs: 1735699104
+  nsecs: 243028641
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699104.743361]: received mask_obs with header seq: 2560
+stamp:
+  secs: 1735699104
+  nsecs: 746979236
+frame_id: '' and grasped_object_mask header seq: 192300
+stamp:
+  secs: 1735699104
+  nsecs: 509132147
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699105.003870]: received mask_obs with header seq: 2561
+stamp:
+  secs: 1735699105
+  nsecs:   7356405
+frame_id: '' and grasped_object_mask header seq: 192308
+stamp:
+  secs: 1735699104
+  nsecs: 775254011
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699105.261817]: received mask_obs with header seq: 2562
+stamp:
+  secs: 1735699105
+  nsecs: 265281677
+frame_id: '' and grasped_object_mask header seq: 192315
+stamp:
+  secs: 1735699105
+  nsecs:   8127689
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699105.519421]: received mask_obs with header seq: 2563
+stamp:
+  secs: 1735699105
+  nsecs: 523038387
+frame_id: '' and grasped_object_mask header seq: 192323
+stamp:
+  secs: 1735699105
+  nsecs: 274249315
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699105.777193]: received mask_obs with header seq: 2564
+stamp:
+  secs: 1735699105
+  nsecs: 780673503
+frame_id: '' and grasped_object_mask header seq: 192331
+stamp:
+  secs: 1735699105
+  nsecs: 540349722
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699106.036361]: received mask_obs with header seq: 2565
+stamp:
+  secs: 1735699106
+  nsecs:  39885282
+frame_id: '' and grasped_object_mask header seq: 192339
+stamp:
+  secs: 1735699105
+  nsecs: 807533979
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699106.307182]: received mask_obs with header seq: 2566
+stamp:
+  secs: 1735699106
+  nsecs: 310754299
+frame_id: '' and grasped_object_mask header seq: 192347
+stamp:
+  secs: 1735699106
+  nsecs:  75503826
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699106.575279]: received mask_obs with header seq: 2567
+stamp:
+  secs: 1735699106
+  nsecs: 578902244
+frame_id: '' and grasped_object_mask header seq: 192355
+stamp:
+  secs: 1735699106
+  nsecs: 343626976
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699106.836144]: received mask_obs with header seq: 2568
+stamp:
+  secs: 1735699106
+  nsecs: 839780807
+frame_id: '' and grasped_object_mask header seq: 192363
+stamp:
+  secs: 1735699106
+  nsecs: 611904860
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699107.101593]: received mask_obs with header seq: 2569
+stamp:
+  secs: 1735699107
+  nsecs: 104794979
+frame_id: '' and grasped_object_mask header seq: 192371
+stamp:
+  secs: 1735699106
+  nsecs: 878601551
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699107.350091]: received mask_obs with header seq: 2570
+stamp:
+  secs: 1735699107
+  nsecs: 353633165
+frame_id: '' and grasped_object_mask header seq: 192378
+stamp:
+  secs: 1735699107
+  nsecs: 111753702
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699107.608387]: received mask_obs with header seq: 2571
+stamp:
+  secs: 1735699107
+  nsecs: 611885309
+frame_id: '' and grasped_object_mask header seq: 192386
+stamp:
+  secs: 1735699107
+  nsecs: 378213644
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699107.868159]: received mask_obs with header seq: 2572
+stamp:
+  secs: 1735699107
+  nsecs: 871854066
+frame_id: '' and grasped_object_mask header seq: 192394
+stamp:
+  secs: 1735699107
+  nsecs: 644674063
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699108.137702]: received mask_obs with header seq: 2573
+stamp:
+  secs: 1735699108
+  nsecs: 139954090
+frame_id: '' and grasped_object_mask header seq: 192402
+stamp:
+  secs: 1735699107
+  nsecs: 911124706
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699108.386102]: received mask_obs with header seq: 2574
+stamp:
+  secs: 1735699108
+  nsecs: 389455795
+frame_id: '' and grasped_object_mask header seq: 192409
+stamp:
+  secs: 1735699108
+  nsecs: 144267321
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699108.666665]: received mask_obs with header seq: 2575
+stamp:
+  secs: 1735699108
+  nsecs: 668689727
+frame_id: '' and grasped_object_mask header seq: 192417
+stamp:
+  secs: 1735699108
+  nsecs: 410716534
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699108.920511]: received mask_obs with header seq: 2576
+stamp:
+  secs: 1735699108
+  nsecs: 924171686
+frame_id: '' and grasped_object_mask header seq: 192425
+stamp:
+  secs: 1735699108
+  nsecs: 677166224
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699109.174346]: received mask_obs with header seq: 2577
+stamp:
+  secs: 1735699109
+  nsecs: 177667140
+frame_id: '' and grasped_object_mask header seq: 192433
+stamp:
+  secs: 1735699108
+  nsecs: 943611622
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699109.435047]: received mask_obs with header seq: 2578
+stamp:
+  secs: 1735699109
+  nsecs: 438718795
+frame_id: '' and grasped_object_mask header seq: 192441
+stamp:
+  secs: 1735699109
+  nsecs: 210056543
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699109.701685]: received mask_obs with header seq: 2579
+stamp:
+  secs: 1735699109
+  nsecs: 705315351
+frame_id: '' and grasped_object_mask header seq: 192449
+stamp:
+  secs: 1735699109
+  nsecs: 476501465
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699109.954939]: received mask_obs with header seq: 2580
+stamp:
+  secs: 1735699109
+  nsecs: 958501577
+frame_id: '' and grasped_object_mask header seq: 192456
+stamp:
+  secs: 1735699109
+  nsecs: 709642887
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699110.209764]: received mask_obs with header seq: 2581
+stamp:
+  secs: 1735699110
+  nsecs: 213425636
+frame_id: '' and grasped_object_mask header seq: 192464
+stamp:
+  secs: 1735699109
+  nsecs: 976095676
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699110.468887]: received mask_obs with header seq: 2582
+stamp:
+  secs: 1735699110
+  nsecs: 471864223
+frame_id: '' and grasped_object_mask header seq: 192472
+stamp:
+  secs: 1735699110
+  nsecs: 242548466
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699110.734986]: received mask_obs with header seq: 2583
+stamp:
+  secs: 1735699110
+  nsecs: 738434076
+frame_id: '' and grasped_object_mask header seq: 192480
+stamp:
+  secs: 1735699110
+  nsecs: 509002447
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699110.996569]: received mask_obs with header seq: 2584
+stamp:
+  secs: 1735699111
+  nsecs:    208377
+frame_id: '' and grasped_object_mask header seq: 192488
+stamp:
+  secs: 1735699110
+  nsecs: 775454283
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699111.263597]: received mask_obs with header seq: 2585
+stamp:
+  secs: 1735699111
+  nsecs: 267169713
+frame_id: '' and grasped_object_mask header seq: 192495
+stamp:
+  secs: 1735699111
+  nsecs:   8599043
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699111.533491]: received mask_obs with header seq: 2586
+stamp:
+  secs: 1735699111
+  nsecs: 537139415
+frame_id: '' and grasped_object_mask header seq: 192503
+stamp:
+  secs: 1735699111
+  nsecs: 275049448
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699111.803350]: received mask_obs with header seq: 2587
+stamp:
+  secs: 1735699111
+  nsecs: 806672334
+frame_id: '' and grasped_object_mask header seq: 192512
+stamp:
+  secs: 1735699111
+  nsecs: 574805975
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699112.066325]: received mask_obs with header seq: 2588
+stamp:
+  secs: 1735699112
+  nsecs:  69916009
+frame_id: '' and grasped_object_mask header seq: 192520
+stamp:
+  secs: 1735699111
+  nsecs: 841259241
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699112.330996]: received mask_obs with header seq: 2589
+stamp:
+  secs: 1735699112
+  nsecs: 334536552
+frame_id: '' and grasped_object_mask header seq: 192528
+stamp:
+  secs: 1735699112
+  nsecs: 107712507
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699112.582285]: received mask_obs with header seq: 2590
+stamp:
+  secs: 1735699112
+  nsecs: 585886955
+frame_id: '' and grasped_object_mask header seq: 192535
+stamp:
+  secs: 1735699112
+  nsecs: 340859413
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699112.837386]: received mask_obs with header seq: 2591
+stamp:
+  secs: 1735699112
+  nsecs: 840974092
+frame_id: '' and grasped_object_mask header seq: 192543
+stamp:
+  secs: 1735699112
+  nsecs: 607312441
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699113.099837]: received mask_obs with header seq: 2592
+stamp:
+  secs: 1735699113
+  nsecs: 103209018
+frame_id: '' and grasped_object_mask header seq: 192551
+stamp:
+  secs: 1735699112
+  nsecs: 873765469
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699113.366292]: received mask_obs with header seq: 2593
+stamp:
+  secs: 1735699113
+  nsecs: 369778156
+frame_id: '' and grasped_object_mask header seq: 192559
+stamp:
+  secs: 1735699113
+  nsecs: 140218019
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699113.630641]: received mask_obs with header seq: 2594
+stamp:
+  secs: 1735699113
+  nsecs: 634176015
+frame_id: '' and grasped_object_mask header seq: 192567
+stamp:
+  secs: 1735699113
+  nsecs: 406670570
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699113.881262]: received mask_obs with header seq: 2595
+stamp:
+  secs: 1735699113
+  nsecs: 884904623
+frame_id: '' and grasped_object_mask header seq: 192574
+stamp:
+  secs: 1735699113
+  nsecs: 639817476
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699114.137567]: received mask_obs with header seq: 2596
+stamp:
+  secs: 1735699114
+  nsecs: 141066551
+frame_id: '' and grasped_object_mask header seq: 192582
+stamp:
+  secs: 1735699113
+  nsecs: 906272173
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699114.399365]: received mask_obs with header seq: 2597
+stamp:
+  secs: 1735699114
+  nsecs: 402976989
+frame_id: '' and grasped_object_mask header seq: 192590
+stamp:
+  secs: 1735699114
+  nsecs: 172728300
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699114.666530]: received mask_obs with header seq: 2598
+stamp:
+  secs: 1735699114
+  nsecs: 669038295
+frame_id: '' and grasped_object_mask header seq: 192598
+stamp:
+  secs: 1735699114
+  nsecs: 439183712
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699114.931584]: received mask_obs with header seq: 2599
+stamp:
+  secs: 1735699114
+  nsecs: 935168027
+frame_id: '' and grasped_object_mask header seq: 192605
+stamp:
+  secs: 1735699114
+  nsecs: 672329664
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699115.197352]: received mask_obs with header seq: 2600
+stamp:
+  secs: 1735699115
+  nsecs: 200808525
+frame_id: '' and grasped_object_mask header seq: 192614
+stamp:
+  secs: 1735699114
+  nsecs: 972074509
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699115.459555]: received mask_obs with header seq: 2601
+stamp:
+  secs: 1735699115
+  nsecs: 463177680
+frame_id: '' and grasped_object_mask header seq: 192622
+stamp:
+  secs: 1735699115
+  nsecs: 238513231
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699115.711519]: received mask_obs with header seq: 2602
+stamp:
+  secs: 1735699115
+  nsecs: 714768171
+frame_id: '' and grasped_object_mask header seq: 192629
+stamp:
+  secs: 1735699115
+  nsecs: 471647024
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699115.970563]: received mask_obs with header seq: 2603
+stamp:
+  secs: 1735699115
+  nsecs: 974035739
+frame_id: '' and grasped_object_mask header seq: 192637
+stamp:
+  secs: 1735699115
+  nsecs: 738090277
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699116.240451]: received mask_obs with header seq: 2604
+stamp:
+  secs: 1735699116
+  nsecs: 243978261
+frame_id: '' and grasped_object_mask header seq: 192645
+stamp:
+  secs: 1735699116
+  nsecs:   4544020
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699116.501370]: received mask_obs with header seq: 2605
+stamp:
+  secs: 1735699116
+  nsecs: 505037784
+frame_id: '' and grasped_object_mask header seq: 192653
+stamp:
+  secs: 1735699116
+  nsecs: 270996809
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699116.766564]: received mask_obs with header seq: 2606
+stamp:
+  secs: 1735699116
+  nsecs: 770151853
+frame_id: '' and grasped_object_mask header seq: 192661
+stamp:
+  secs: 1735699116
+  nsecs: 537449598
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699117.029272]: received mask_obs with header seq: 2607
+stamp:
+  secs: 1735699117
+  nsecs:  32633304
+frame_id: '' and grasped_object_mask header seq: 192669
+stamp:
+  secs: 1735699116
+  nsecs: 803910971
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699117.286725]: received mask_obs with header seq: 2608
+stamp:
+  secs: 1735699117
+  nsecs: 288367509
+frame_id: '' and grasped_object_mask header seq: 192676
+stamp:
+  secs: 1735699117
+  nsecs:  37070036
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699117.552938]: received mask_obs with header seq: 2609
+stamp:
+  secs: 1735699117
+  nsecs: 555431365
+frame_id: '' and grasped_object_mask header seq: 192684
+stamp:
+  secs: 1735699117
+  nsecs: 303537607
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699117.824477]: received mask_obs with header seq: 2610
+stamp:
+  secs: 1735699117
+  nsecs: 828063964
+frame_id: '' and grasped_object_mask header seq: 192692
+stamp:
+  secs: 1735699117
+  nsecs: 570006132
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699118.077042]: received mask_obs with header seq: 2611
+stamp:
+  secs: 1735699118
+  nsecs:  80387115
+frame_id: '' and grasped_object_mask header seq: 192700
+stamp:
+  secs: 1735699117
+  nsecs: 836455345
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699118.336781]: received mask_obs with header seq: 2612
+stamp:
+  secs: 1735699118
+  nsecs: 340374946
+frame_id: '' and grasped_object_mask header seq: 192708
+stamp:
+  secs: 1735699118
+  nsecs: 102893591
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699118.598344]: received mask_obs with header seq: 2613
+stamp:
+  secs: 1735699118
+  nsecs: 601634979
+frame_id: '' and grasped_object_mask header seq: 192716
+stamp:
+  secs: 1735699118
+  nsecs: 369331121
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699118.860285]: received mask_obs with header seq: 2614
+stamp:
+  secs: 1735699118
+  nsecs: 863959550
+frame_id: '' and grasped_object_mask header seq: 192724
+stamp:
+  secs: 1735699118
+  nsecs: 635767460
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699119.126751]: received mask_obs with header seq: 2615
+stamp:
+  secs: 1735699119
+  nsecs: 130401372
+frame_id: '' and grasped_object_mask header seq: 192732
+stamp:
+  secs: 1735699118
+  nsecs: 902207613
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699119.373505]: received mask_obs with header seq: 2616
+stamp:
+  secs: 1735699119
+  nsecs: 377128124
+frame_id: '' and grasped_object_mask header seq: 192739
+stamp:
+  secs: 1735699119
+  nsecs: 135343313
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699119.648576]: received mask_obs with header seq: 2617
+stamp:
+  secs: 1735699119
+  nsecs: 652101516
+frame_id: '' and grasped_object_mask header seq: 192747
+stamp:
+  secs: 1735699119
+  nsecs: 401782990
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699119.903231]: received mask_obs with header seq: 2618
+stamp:
+  secs: 1735699119
+  nsecs: 906856298
+frame_id: '' and grasped_object_mask header seq: 192755
+stamp:
+  secs: 1735699119
+  nsecs: 668223143
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699120.161948]: received mask_obs with header seq: 2619
+stamp:
+  secs: 1735699120
+  nsecs: 165580034
+frame_id: '' and grasped_object_mask header seq: 192763
+stamp:
+  secs: 1735699119
+  nsecs: 934669495
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699120.426541]: received mask_obs with header seq: 2620
+stamp:
+  secs: 1735699120
+  nsecs: 429819345
+frame_id: '' and grasped_object_mask header seq: 192771
+stamp:
+  secs: 1735699120
+  nsecs: 201115370
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699120.683775]: received mask_obs with header seq: 2621
+stamp:
+  secs: 1735699120
+  nsecs: 686586141
+frame_id: '' and grasped_object_mask header seq: 192778
+stamp:
+  secs: 1735699120
+  nsecs: 434256315
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699120.955911]: received mask_obs with header seq: 2622
+stamp:
+  secs: 1735699120
+  nsecs: 959160327
+frame_id: '' and grasped_object_mask header seq: 192786
+stamp:
+  secs: 1735699120
+  nsecs: 700704336
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699121.221915]: received mask_obs with header seq: 2623
+stamp:
+  secs: 1735699121
+  nsecs: 225579500
+frame_id: '' and grasped_object_mask header seq: 192794
+stamp:
+  secs: 1735699120
+  nsecs: 967169285
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699121.491251]: received mask_obs with header seq: 2624
+stamp:
+  secs: 1735699121
+  nsecs: 494813203
+frame_id: '' and grasped_object_mask header seq: 192802
+stamp:
+  secs: 1735699121
+  nsecs: 233634710
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699121.764813]: received mask_obs with header seq: 2625
+stamp:
+  secs: 1735699121
+  nsecs: 768293380
+frame_id: '' and grasped_object_mask header seq: 192811
+stamp:
+  secs: 1735699121
+  nsecs: 533410072
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699122.023879]: received mask_obs with header seq: 2626
+stamp:
+  secs: 1735699122
+  nsecs:  27521133
+frame_id: '' and grasped_object_mask header seq: 192819
+stamp:
+  secs: 1735699121
+  nsecs: 799868822
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699122.293410]: received mask_obs with header seq: 2627
+stamp:
+  secs: 1735699122
+  nsecs: 297074079
+frame_id: '' and grasped_object_mask header seq: 192827
+stamp:
+  secs: 1735699122
+  nsecs:  66319227
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699122.558784]: received mask_obs with header seq: 2628
+stamp:
+  secs: 1735699122
+  nsecs: 561771631
+frame_id: '' and grasped_object_mask header seq: 192835
+stamp:
+  secs: 1735699122
+  nsecs: 332770824
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699122.664423]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1735699122.824035]: received mask_obs with header seq: 2629
+stamp:
+  secs: 1735699122
+  nsecs: 824240922
+frame_id: '' and grasped_object_mask header seq: 192843
+stamp:
+  secs: 1735699122
+  nsecs: 599221230
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699123.431471]: Stored episode 4.
+[INFO] [1735699123.442705]: Saving video...
+[INFO] [1735699123.903179]: Video saved!
+Starting episode 4
+[INFO] [1735699123.973602]: resetting environment
+[INFO] [1735699123.999376]: cleared current plan
+[INFO] [1735699123.999673]: moving to home
+[INFO] [1735699127.610599]: reached home
+[INFO] [1735699127.614311]: reset action history
+[INFO] [1735699130.421558]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735699146.080164]: resetting environment
+[INFO] [1735699146.111493]: cleared current plan
+[INFO] [1735699146.113533]: moving to home
+[INFO] [1735699147.117801]: reached home
+[INFO] [1735699147.127237]: reset action history
+[INFO] [1735699149.863585]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/episode_rosbags/episode_4_2024-12-31-21-39-09.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1748029.log
+started roslaunch server http://158.130.50.37:40195/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [1749134]
+process[print_text-14]: started with pid [1749135]
+process[rosbag_record-15]: started with pid [1749136]
+started bagging!
+[INFO] [1735699150.815922]: received mask_obs with header seq: 2630
+stamp:
+  secs: 1735699150
+  nsecs: 819272041
+frame_id: '' and grasped_object_mask header seq: 193684
+stamp:
+  secs: 1735699150
+  nsecs: 617398739
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699151.044689]: received mask_obs with header seq: 2631
+stamp:
+  secs: 1735699151
+  nsecs:  48195123
+frame_id: '' and grasped_object_mask header seq: 193691
+stamp:
+  secs: 1735699150
+  nsecs: 850253344
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699151.279564]: received mask_obs with header seq: 2632
+stamp:
+  secs: 1735699151
+  nsecs: 282962322
+frame_id: '' and grasped_object_mask header seq: 193698
+stamp:
+  secs: 1735699151
+  nsecs:  83126068
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699151.520903]: received mask_obs with header seq: 2633
+stamp:
+  secs: 1735699151
+  nsecs: 524507999
+frame_id: '' and grasped_object_mask header seq: 193705
+stamp:
+  secs: 1735699151
+  nsecs: 315981388
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699151.762625]: received mask_obs with header seq: 2634
+stamp:
+  secs: 1735699151
+  nsecs: 766040325
+frame_id: '' and grasped_object_mask header seq: 193712
+stamp:
+  secs: 1735699151
+  nsecs: 548821926
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699152.025960]: received mask_obs with header seq: 2635
+stamp:
+  secs: 1735699152
+  nsecs:  27243614
+frame_id: '' and grasped_object_mask header seq: 193719
+stamp:
+  secs: 1735699151
+  nsecs: 781663179
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699152.285016]: received mask_obs with header seq: 2636
+stamp:
+  secs: 1735699152
+  nsecs: 288643836
+frame_id: '' and grasped_object_mask header seq: 193727
+stamp:
+  secs: 1735699152
+  nsecs:  47819853
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699152.544535]: received mask_obs with header seq: 2637
+stamp:
+  secs: 1735699152
+  nsecs: 548141479
+frame_id: '' and grasped_object_mask header seq: 193735
+stamp:
+  secs: 1735699152
+  nsecs: 313956261
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699152.804628]: received mask_obs with header seq: 2638
+stamp:
+  secs: 1735699152
+  nsecs: 808312177
+frame_id: '' and grasped_object_mask header seq: 193743
+stamp:
+  secs: 1735699152
+  nsecs: 580072403
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699153.072371]: received mask_obs with header seq: 2639
+stamp:
+  secs: 1735699153
+  nsecs:  75872182
+frame_id: '' and grasped_object_mask header seq: 193751
+stamp:
+  secs: 1735699152
+  nsecs: 846201897
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699153.336763]: received mask_obs with header seq: 2640
+stamp:
+  secs: 1735699153
+  nsecs: 340443611
+frame_id: '' and grasped_object_mask header seq: 193759
+stamp:
+  secs: 1735699153
+  nsecs: 112355947
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699153.605675]: received mask_obs with header seq: 2641
+stamp:
+  secs: 1735699153
+  nsecs: 609309911
+frame_id: '' and grasped_object_mask header seq: 193767
+stamp:
+  secs: 1735699153
+  nsecs: 378489256
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699153.867148]: received mask_obs with header seq: 2642
+stamp:
+  secs: 1735699153
+  nsecs: 870795249
+frame_id: '' and grasped_object_mask header seq: 193775
+stamp:
+  secs: 1735699153
+  nsecs: 644602537
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699154.131860]: received mask_obs with header seq: 2643
+stamp:
+  secs: 1735699154
+  nsecs: 135498523
+frame_id: '' and grasped_object_mask header seq: 193783
+stamp:
+  secs: 1735699153
+  nsecs: 910742760
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699154.381428]: received mask_obs with header seq: 2644
+stamp:
+  secs: 1735699154
+  nsecs: 384903192
+frame_id: '' and grasped_object_mask header seq: 193790
+stamp:
+  secs: 1735699154
+  nsecs: 143618345
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699154.642533]: received mask_obs with header seq: 2645
+stamp:
+  secs: 1735699154
+  nsecs: 646194934
+frame_id: '' and grasped_object_mask header seq: 193798
+stamp:
+  secs: 1735699154
+  nsecs: 409742117
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699154.895899]: received mask_obs with header seq: 2646
+stamp:
+  secs: 1735699154
+  nsecs: 899514436
+frame_id: '' and grasped_object_mask header seq: 193806
+stamp:
+  secs: 1735699154
+  nsecs: 675845623
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699155.148555]: received mask_obs with header seq: 2647
+stamp:
+  secs: 1735699155
+  nsecs: 152189731
+frame_id: '' and grasped_object_mask header seq: 193813
+stamp:
+  secs: 1735699154
+  nsecs: 908722162
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699155.403667]: received mask_obs with header seq: 2648
+stamp:
+  secs: 1735699155
+  nsecs: 407336950
+frame_id: '' and grasped_object_mask header seq: 193821
+stamp:
+  secs: 1735699155
+  nsecs: 174864292
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699155.666686]: received mask_obs with header seq: 2649
+stamp:
+  secs: 1735699155
+  nsecs: 669870376
+frame_id: '' and grasped_object_mask header seq: 193829
+stamp:
+  secs: 1735699155
+  nsecs: 440987110
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699155.929509]: received mask_obs with header seq: 2650
+stamp:
+  secs: 1735699155
+  nsecs: 933120965
+frame_id: '' and grasped_object_mask header seq: 193837
+stamp:
+  secs: 1735699155
+  nsecs: 707088470
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699156.180080]: received mask_obs with header seq: 2651
+stamp:
+  secs: 1735699156
+  nsecs: 183726072
+frame_id: '' and grasped_object_mask header seq: 193844
+stamp:
+  secs: 1735699155
+  nsecs: 939972401
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699156.436425]: received mask_obs with header seq: 2652
+stamp:
+  secs: 1735699156
+  nsecs: 439865827
+frame_id: '' and grasped_object_mask header seq: 193852
+stamp:
+  secs: 1735699156
+  nsecs: 206115246
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699156.697699]: received mask_obs with header seq: 2653
+stamp:
+  secs: 1735699156
+  nsecs: 701345205
+frame_id: '' and grasped_object_mask header seq: 193860
+stamp:
+  secs: 1735699156
+  nsecs: 472236395
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699156.965343]: received mask_obs with header seq: 2654
+stamp:
+  secs: 1735699156
+  nsecs: 968901157
+frame_id: '' and grasped_object_mask header seq: 193868
+stamp:
+  secs: 1735699156
+  nsecs: 738340616
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699157.215423]: received mask_obs with header seq: 2655
+stamp:
+  secs: 1735699157
+  nsecs: 218894958
+frame_id: '' and grasped_object_mask header seq: 193875
+stamp:
+  secs: 1735699156
+  nsecs: 971225023
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699157.473559]: received mask_obs with header seq: 2656
+stamp:
+  secs: 1735699157
+  nsecs: 477141141
+frame_id: '' and grasped_object_mask header seq: 193883
+stamp:
+  secs: 1735699157
+  nsecs: 237358570
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699157.742648]: received mask_obs with header seq: 2657
+stamp:
+  secs: 1735699157
+  nsecs: 746238231
+frame_id: '' and grasped_object_mask header seq: 193891
+stamp:
+  secs: 1735699157
+  nsecs: 503471613
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699158.001706]: received mask_obs with header seq: 2658
+stamp:
+  secs: 1735699158
+  nsecs:   5067586
+frame_id: '' and grasped_object_mask header seq: 193899
+stamp:
+  secs: 1735699157
+  nsecs: 769577503
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699158.266209]: received mask_obs with header seq: 2659
+stamp:
+  secs: 1735699158
+  nsecs: 268987894
+frame_id: '' and grasped_object_mask header seq: 193907
+stamp:
+  secs: 1735699158
+  nsecs:  35728931
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699158.525451]: received mask_obs with header seq: 2660
+stamp:
+  secs: 1735699158
+  nsecs: 529053926
+frame_id: '' and grasped_object_mask header seq: 193915
+stamp:
+  secs: 1735699158
+  nsecs: 301858902
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699158.782058]: received mask_obs with header seq: 2661
+stamp:
+  secs: 1735699158
+  nsecs: 785625696
+frame_id: '' and grasped_object_mask header seq: 193922
+stamp:
+  secs: 1735699158
+  nsecs: 534706116
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699159.041370]: received mask_obs with header seq: 2662
+stamp:
+  secs: 1735699159
+  nsecs:  44977426
+frame_id: '' and grasped_object_mask header seq: 193930
+stamp:
+  secs: 1735699158
+  nsecs: 800818920
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699159.300115]: received mask_obs with header seq: 2663
+stamp:
+  secs: 1735699159
+  nsecs: 303654670
+frame_id: '' and grasped_object_mask header seq: 193938
+stamp:
+  secs: 1735699159
+  nsecs:  66964388
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699159.563071]: received mask_obs with header seq: 2664
+stamp:
+  secs: 1735699159
+  nsecs: 566609382
+frame_id: '' and grasped_object_mask header seq: 193946
+stamp:
+  secs: 1735699159
+  nsecs: 333088875
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699159.833918]: received mask_obs with header seq: 2665
+stamp:
+  secs: 1735699159
+  nsecs: 837558031
+frame_id: '' and grasped_object_mask header seq: 193954
+stamp:
+  secs: 1735699159
+  nsecs: 599192619
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699160.099641]: received mask_obs with header seq: 2666
+stamp:
+  secs: 1735699160
+  nsecs: 103258371
+frame_id: '' and grasped_object_mask header seq: 193962
+stamp:
+  secs: 1735699159
+  nsecs: 865315914
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699160.368861]: received mask_obs with header seq: 2667
+stamp:
+  secs: 1735699160
+  nsecs: 371618509
+frame_id: '' and grasped_object_mask header seq: 193970
+stamp:
+  secs: 1735699160
+  nsecs: 131450415
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699160.635381]: received mask_obs with header seq: 2668
+stamp:
+  secs: 1735699160
+  nsecs: 638908624
+frame_id: '' and grasped_object_mask header seq: 193978
+stamp:
+  secs: 1735699160
+  nsecs: 397563934
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699160.899604]: received mask_obs with header seq: 2669
+stamp:
+  secs: 1735699160
+  nsecs: 903215408
+frame_id: '' and grasped_object_mask header seq: 193986
+stamp:
+  secs: 1735699160
+  nsecs: 663656235
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699161.170495]: received mask_obs with header seq: 2670
+stamp:
+  secs: 1735699161
+  nsecs: 174053430
+frame_id: '' and grasped_object_mask header seq: 193994
+stamp:
+  secs: 1735699160
+  nsecs: 929797411
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699161.428707]: received mask_obs with header seq: 2671
+stamp:
+  secs: 1735699161
+  nsecs: 432303667
+frame_id: '' and grasped_object_mask header seq: 194002
+stamp:
+  secs: 1735699161
+  nsecs: 195937634
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699161.698519]: received mask_obs with header seq: 2672
+stamp:
+  secs: 1735699161
+  nsecs: 698583126
+frame_id: '' and grasped_object_mask header seq: 194010
+stamp:
+  secs: 1735699161
+  nsecs: 462056637
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699161.956081]: received mask_obs with header seq: 2673
+stamp:
+  secs: 1735699161
+  nsecs: 959690570
+frame_id: '' and grasped_object_mask header seq: 194018
+stamp:
+  secs: 1735699161
+  nsecs: 728156567
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699162.222949]: received mask_obs with header seq: 2674
+stamp:
+  secs: 1735699162
+  nsecs: 226077079
+frame_id: '' and grasped_object_mask header seq: 194025
+stamp:
+  secs: 1735699161
+  nsecs: 962551594
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699162.474472]: received mask_obs with header seq: 2675
+stamp:
+  secs: 1735699162
+  nsecs: 477813959
+frame_id: '' and grasped_object_mask header seq: 194033
+stamp:
+  secs: 1735699162
+  nsecs: 230576992
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699162.736149]: received mask_obs with header seq: 2676
+stamp:
+  secs: 1735699162
+  nsecs: 737695455
+frame_id: '' and grasped_object_mask header seq: 194041
+stamp:
+  secs: 1735699162
+  nsecs: 498758078
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699162.996508]: received mask_obs with header seq: 2677
+stamp:
+  secs: 1735699162
+  nsecs: 999735116
+frame_id: '' and grasped_object_mask header seq: 194049
+stamp:
+  secs: 1735699162
+  nsecs: 766851425
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699163.259871]: received mask_obs with header seq: 2678
+stamp:
+  secs: 1735699163
+  nsecs: 262999534
+frame_id: '' and grasped_object_mask header seq: 194057
+stamp:
+  secs: 1735699163
+  nsecs:  33309460
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699163.527733]: received mask_obs with header seq: 2679
+stamp:
+  secs: 1735699163
+  nsecs: 529783725
+frame_id: '' and grasped_object_mask header seq: 194065
+stamp:
+  secs: 1735699163
+  nsecs: 299767017
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699163.791370]: received mask_obs with header seq: 2680
+stamp:
+  secs: 1735699163
+  nsecs: 794692754
+frame_id: '' and grasped_object_mask header seq: 194073
+stamp:
+  secs: 1735699163
+  nsecs: 566225052
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699164.055724]: received mask_obs with header seq: 2681
+stamp:
+  secs: 1735699164
+  nsecs:  59136629
+frame_id: '' and grasped_object_mask header seq: 194081
+stamp:
+  secs: 1735699163
+  nsecs: 832678318
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699164.321494]: received mask_obs with header seq: 2682
+stamp:
+  secs: 1735699164
+  nsecs: 325083971
+frame_id: '' and grasped_object_mask header seq: 194088
+stamp:
+  secs: 1735699164
+  nsecs:  65817595
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699164.575624]: received mask_obs with header seq: 2683
+stamp:
+  secs: 1735699164
+  nsecs: 578987836
+frame_id: '' and grasped_object_mask header seq: 194096
+stamp:
+  secs: 1735699164
+  nsecs: 332261562
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699164.834512]: received mask_obs with header seq: 2684
+stamp:
+  secs: 1735699164
+  nsecs: 838179111
+frame_id: '' and grasped_object_mask header seq: 194104
+stamp:
+  secs: 1735699164
+  nsecs: 598704815
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699165.095633]: received mask_obs with header seq: 2685
+stamp:
+  secs: 1735699165
+  nsecs:  99159240
+frame_id: '' and grasped_object_mask header seq: 194112
+stamp:
+  secs: 1735699164
+  nsecs: 865151167
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699165.356665]: received mask_obs with header seq: 2686
+stamp:
+  secs: 1735699165
+  nsecs: 360090017
+frame_id: '' and grasped_object_mask header seq: 194120
+stamp:
+  secs: 1735699165
+  nsecs: 131599188
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699165.621289]: received mask_obs with header seq: 2687
+stamp:
+  secs: 1735699165
+  nsecs: 624719142
+frame_id: '' and grasped_object_mask header seq: 194128
+stamp:
+  secs: 1735699165
+  nsecs: 398046732
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699165.874479]: received mask_obs with header seq: 2688
+stamp:
+  secs: 1735699165
+  nsecs: 878144025
+frame_id: '' and grasped_object_mask header seq: 194135
+stamp:
+  secs: 1735699165
+  nsecs: 631188393
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699166.132308]: received mask_obs with header seq: 2689
+stamp:
+  secs: 1735699166
+  nsecs: 135494709
+frame_id: '' and grasped_object_mask header seq: 194143
+stamp:
+  secs: 1735699165
+  nsecs: 897628307
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699166.391679]: received mask_obs with header seq: 2690
+stamp:
+  secs: 1735699166
+  nsecs: 395318746
+frame_id: '' and grasped_object_mask header seq: 194151
+stamp:
+  secs: 1735699166
+  nsecs: 164063454
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699166.657135]: received mask_obs with header seq: 2691
+stamp:
+  secs: 1735699166
+  nsecs: 660754919
+frame_id: '' and grasped_object_mask header seq: 194159
+stamp:
+  secs: 1735699166
+  nsecs: 430497885
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699166.917318]: received mask_obs with header seq: 2692
+stamp:
+  secs: 1735699166
+  nsecs: 920724153
+frame_id: '' and grasped_object_mask header seq: 194167
+stamp:
+  secs: 1735699166
+  nsecs: 696931839
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699167.171360]: received mask_obs with header seq: 2693
+stamp:
+  secs: 1735699167
+  nsecs: 174676418
+frame_id: '' and grasped_object_mask header seq: 194174
+stamp:
+  secs: 1735699166
+  nsecs: 930068254
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699167.427813]: received mask_obs with header seq: 2694
+stamp:
+  secs: 1735699167
+  nsecs: 431086063
+frame_id: '' and grasped_object_mask header seq: 194182
+stamp:
+  secs: 1735699167
+  nsecs: 196512699
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699167.687915]: received mask_obs with header seq: 2695
+stamp:
+  secs: 1735699167
+  nsecs: 691081523
+frame_id: '' and grasped_object_mask header seq: 194190
+stamp:
+  secs: 1735699167
+  nsecs: 462955236
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699167.954173]: received mask_obs with header seq: 2696
+stamp:
+  secs: 1735699167
+  nsecs: 957466363
+frame_id: '' and grasped_object_mask header seq: 194198
+stamp:
+  secs: 1735699167
+  nsecs: 729397535
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699168.213247]: received mask_obs with header seq: 2697
+stamp:
+  secs: 1735699168
+  nsecs: 216699123
+frame_id: '' and grasped_object_mask header seq: 194205
+stamp:
+  secs: 1735699167
+  nsecs: 962548018
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699168.463559]: received mask_obs with header seq: 2698
+stamp:
+  secs: 1735699168
+  nsecs: 466953039
+frame_id: '' and grasped_object_mask header seq: 194213
+stamp:
+  secs: 1735699168
+  nsecs: 229005575
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699168.722076]: received mask_obs with header seq: 2699
+stamp:
+  secs: 1735699168
+  nsecs: 725666522
+frame_id: '' and grasped_object_mask header seq: 194221
+stamp:
+  secs: 1735699168
+  nsecs: 495463848
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699168.988518]: received mask_obs with header seq: 2700
+stamp:
+  secs: 1735699168
+  nsecs: 991945505
+frame_id: '' and grasped_object_mask header seq: 194229
+stamp:
+  secs: 1735699168
+  nsecs: 761921883
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699169.250775]: received mask_obs with header seq: 2701
+stamp:
+  secs: 1735699169
+  nsecs: 253774881
+frame_id: '' and grasped_object_mask header seq: 194237
+stamp:
+  secs: 1735699169
+  nsecs:  28377771
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699169.503711]: received mask_obs with header seq: 2702
+stamp:
+  secs: 1735699169
+  nsecs: 507259845
+frame_id: '' and grasped_object_mask header seq: 194244
+stamp:
+  secs: 1735699169
+  nsecs: 261525631
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699169.757032]: received mask_obs with header seq: 2703
+stamp:
+  secs: 1735699169
+  nsecs: 760664463
+frame_id: '' and grasped_object_mask header seq: 194252
+stamp:
+  secs: 1735699169
+  nsecs: 527981281
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699170.019043]: received mask_obs with header seq: 2704
+stamp:
+  secs: 1735699170
+  nsecs:  22353410
+frame_id: '' and grasped_object_mask header seq: 194260
+stamp:
+  secs: 1735699169
+  nsecs: 794437647
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699170.283501]: received mask_obs with header seq: 2705
+stamp:
+  secs: 1735699170
+  nsecs: 286826133
+frame_id: '' and grasped_object_mask header seq: 194268
+stamp:
+  secs: 1735699170
+  nsecs:  60893774
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699170.537384]: received mask_obs with header seq: 2706
+stamp:
+  secs: 1735699170
+  nsecs: 540989160
+frame_id: '' and grasped_object_mask header seq: 194275
+stamp:
+  secs: 1735699170
+  nsecs: 294043541
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699170.792162]: received mask_obs with header seq: 2707
+stamp:
+  secs: 1735699170
+  nsecs: 795655012
+frame_id: '' and grasped_object_mask header seq: 194283
+stamp:
+  secs: 1735699170
+  nsecs: 560499668
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699171.053667]: received mask_obs with header seq: 2708
+stamp:
+  secs: 1735699171
+  nsecs:  57061672
+frame_id: '' and grasped_object_mask header seq: 194291
+stamp:
+  secs: 1735699170
+  nsecs: 826957464
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699171.316219]: received mask_obs with header seq: 2709
+stamp:
+  secs: 1735699171
+  nsecs: 319726943
+frame_id: '' and grasped_object_mask header seq: 194299
+stamp:
+  secs: 1735699171
+  nsecs:  93417645
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699171.575229]: received mask_obs with header seq: 2710
+stamp:
+  secs: 1735699171
+  nsecs: 577046632
+frame_id: '' and grasped_object_mask header seq: 194306
+stamp:
+  secs: 1735699171
+  nsecs: 326570988
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699171.828087]: received mask_obs with header seq: 2711
+stamp:
+  secs: 1735699171
+  nsecs: 831703662
+frame_id: '' and grasped_object_mask header seq: 194314
+stamp:
+  secs: 1735699171
+  nsecs: 593031883
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699172.090149]: received mask_obs with header seq: 2712
+stamp:
+  secs: 1735699172
+  nsecs:  93705892
+frame_id: '' and grasped_object_mask header seq: 194322
+stamp:
+  secs: 1735699171
+  nsecs: 859488964
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699172.350699]: received mask_obs with header seq: 2713
+stamp:
+  secs: 1735699172
+  nsecs: 354309082
+frame_id: '' and grasped_object_mask header seq: 194330
+stamp:
+  secs: 1735699172
+  nsecs: 125942945
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699172.615762]: received mask_obs with header seq: 2714
+stamp:
+  secs: 1735699172
+  nsecs: 618750095
+frame_id: '' and grasped_object_mask header seq: 194338
+stamp:
+  secs: 1735699172
+  nsecs: 392395973
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699172.883662]: received mask_obs with header seq: 2715
+stamp:
+  secs: 1735699172
+  nsecs: 886990785
+frame_id: '' and grasped_object_mask header seq: 194345
+stamp:
+  secs: 1735699172
+  nsecs: 625541925
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699173.153741]: received mask_obs with header seq: 2716
+stamp:
+  secs: 1735699173
+  nsecs: 157084226
+frame_id: '' and grasped_object_mask header seq: 194354
+stamp:
+  secs: 1735699172
+  nsecs: 925306320
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699173.401143]: received mask_obs with header seq: 2717
+stamp:
+  secs: 1735699173
+  nsecs: 404433727
+frame_id: '' and grasped_object_mask header seq: 194361
+stamp:
+  secs: 1735699173
+  nsecs: 158458233
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699173.668947]: received mask_obs with header seq: 2718
+stamp:
+  secs: 1735699173
+  nsecs: 672609806
+frame_id: '' and grasped_object_mask header seq: 194369
+stamp:
+  secs: 1735699173
+  nsecs: 424917459
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699173.924432]: received mask_obs with header seq: 2719
+stamp:
+  secs: 1735699173
+  nsecs: 928059577
+frame_id: '' and grasped_object_mask header seq: 194377
+stamp:
+  secs: 1735699173
+  nsecs: 691376925
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699174.185628]: received mask_obs with header seq: 2720
+stamp:
+  secs: 1735699174
+  nsecs: 188633203
+frame_id: '' and grasped_object_mask header seq: 194385
+stamp:
+  secs: 1735699173
+  nsecs: 957827806
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699174.447221]: received mask_obs with header seq: 2721
+stamp:
+  secs: 1735699174
+  nsecs: 450888156
+frame_id: '' and grasped_object_mask header seq: 194393
+stamp:
+  secs: 1735699174
+  nsecs: 224276304
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699174.702327]: received mask_obs with header seq: 2722
+stamp:
+  secs: 1735699174
+  nsecs: 705947875
+frame_id: '' and grasped_object_mask header seq: 194400
+stamp:
+  secs: 1735699174
+  nsecs: 457418203
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699174.962493]: received mask_obs with header seq: 2723
+stamp:
+  secs: 1735699174
+  nsecs: 966115236
+frame_id: '' and grasped_object_mask header seq: 194408
+stamp:
+  secs: 1735699174
+  nsecs: 723867655
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699175.217440]: received mask_obs with header seq: 2724
+stamp:
+  secs: 1735699175
+  nsecs: 221066236
+frame_id: '' and grasped_object_mask header seq: 194416
+stamp:
+  secs: 1735699174
+  nsecs: 990315676
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699175.483678]: received mask_obs with header seq: 2725
+stamp:
+  secs: 1735699175
+  nsecs: 487325668
+frame_id: '' and grasped_object_mask header seq: 194424
+stamp:
+  secs: 1735699175
+  nsecs: 256763458
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699175.746349]: received mask_obs with header seq: 2726
+stamp:
+  secs: 1735699175
+  nsecs: 749713182
+frame_id: '' and grasped_object_mask header seq: 194432
+stamp:
+  secs: 1735699175
+  nsecs: 523211002
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699176.019684]: received mask_obs with header seq: 2727
+stamp:
+  secs: 1735699176
+  nsecs:  23163795
+frame_id: '' and grasped_object_mask header seq: 194440
+stamp:
+  secs: 1735699175
+  nsecs: 789657831
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699176.281518]: received mask_obs with header seq: 2728
+stamp:
+  secs: 1735699176
+  nsecs: 284955501
+frame_id: '' and grasped_object_mask header seq: 194448
+stamp:
+  secs: 1735699176
+  nsecs:  56098700
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699176.546167]: received mask_obs with header seq: 2729
+stamp:
+  secs: 1735699176
+  nsecs: 549865484
+frame_id: '' and grasped_object_mask header seq: 194456
+stamp:
+  secs: 1735699176
+  nsecs: 322538853
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699176.816491]: received mask_obs with header seq: 2730
+stamp:
+  secs: 1735699176
+  nsecs: 819499015
+frame_id: '' and grasped_object_mask header seq: 194463
+stamp:
+  secs: 1735699176
+  nsecs: 555674076
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699177.076395]: received mask_obs with header seq: 2731
+stamp:
+  secs: 1735699177
+  nsecs:  80027341
+frame_id: '' and grasped_object_mask header seq: 194472
+stamp:
+  secs: 1735699176
+  nsecs: 855424404
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699177.330279]: received mask_obs with header seq: 2732
+stamp:
+  secs: 1735699177
+  nsecs: 333492755
+frame_id: '' and grasped_object_mask header seq: 194479
+stamp:
+  secs: 1735699177
+  nsecs:  88570833
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699177.586703]: received mask_obs with header seq: 2733
+stamp:
+  secs: 1735699177
+  nsecs: 589888811
+frame_id: '' and grasped_object_mask header seq: 194487
+stamp:
+  secs: 1735699177
+  nsecs: 355023623
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699177.855006]: received mask_obs with header seq: 2734
+stamp:
+  secs: 1735699177
+  nsecs: 858596563
+frame_id: '' and grasped_object_mask header seq: 194495
+stamp:
+  secs: 1735699177
+  nsecs: 621477842
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699178.116300]: received mask_obs with header seq: 2735
+stamp:
+  secs: 1735699178
+  nsecs: 119260311
+frame_id: '' and grasped_object_mask header seq: 194503
+stamp:
+  secs: 1735699177
+  nsecs: 887927294
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699178.381267]: received mask_obs with header seq: 2736
+stamp:
+  secs: 1735699178
+  nsecs: 384295225
+frame_id: '' and grasped_object_mask header seq: 194511
+stamp:
+  secs: 1735699178
+  nsecs: 154374599
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699178.647258]: received mask_obs with header seq: 2737
+stamp:
+  secs: 1735699178
+  nsecs: 649752616
+frame_id: '' and grasped_object_mask header seq: 194519
+stamp:
+  secs: 1735699178
+  nsecs: 420821428
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699178.897223]: received mask_obs with header seq: 2738
+stamp:
+  secs: 1735699178
+  nsecs: 900814771
+frame_id: '' and grasped_object_mask header seq: 194526
+stamp:
+  secs: 1735699178
+  nsecs: 653963089
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699179.158967]: received mask_obs with header seq: 2739
+stamp:
+  secs: 1735699179
+  nsecs: 162410497
+frame_id: '' and grasped_object_mask header seq: 194534
+stamp:
+  secs: 1735699178
+  nsecs: 920414209
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699179.414421]: received mask_obs with header seq: 2740
+stamp:
+  secs: 1735699179
+  nsecs: 418047904
+frame_id: '' and grasped_object_mask header seq: 194542
+stamp:
+  secs: 1735699179
+  nsecs: 186866760
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699179.680330]: received mask_obs with header seq: 2741
+stamp:
+  secs: 1735699179
+  nsecs: 683915376
+frame_id: '' and grasped_object_mask header seq: 194550
+stamp:
+  secs: 1735699179
+  nsecs: 453320265
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699179.941625]: received mask_obs with header seq: 2742
+stamp:
+  secs: 1735699179
+  nsecs: 944743394
+frame_id: '' and grasped_object_mask header seq: 194558
+stamp:
+  secs: 1735699179
+  nsecs: 719773054
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699180.194083]: received mask_obs with header seq: 2743
+stamp:
+  secs: 1735699180
+  nsecs: 197580337
+frame_id: '' and grasped_object_mask header seq: 194565
+stamp:
+  secs: 1735699179
+  nsecs: 952907562
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699180.452420]: received mask_obs with header seq: 2744
+stamp:
+  secs: 1735699180
+  nsecs: 456032276
+frame_id: '' and grasped_object_mask header seq: 194573
+stamp:
+  secs: 1735699180
+  nsecs: 219347715
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699180.713062]: received mask_obs with header seq: 2745
+stamp:
+  secs: 1735699180
+  nsecs: 716650009
+frame_id: '' and grasped_object_mask header seq: 194581
+stamp:
+  secs: 1735699180
+  nsecs: 485786438
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699180.977643]: received mask_obs with header seq: 2746
+stamp:
+  secs: 1735699180
+  nsecs: 981264591
+frame_id: '' and grasped_object_mask header seq: 194589
+stamp:
+  secs: 1735699180
+  nsecs: 752226591
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699181.228218]: received mask_obs with header seq: 2747
+stamp:
+  secs: 1735699181
+  nsecs: 231872558
+frame_id: '' and grasped_object_mask header seq: 194596
+stamp:
+  secs: 1735699180
+  nsecs: 985375643
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699181.490352]: received mask_obs with header seq: 2748
+stamp:
+  secs: 1735699181
+  nsecs: 493967056
+frame_id: '' and grasped_object_mask header seq: 194604
+stamp:
+  secs: 1735699181
+  nsecs: 251831293
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699181.752387]: received mask_obs with header seq: 2749
+stamp:
+  secs: 1735699181
+  nsecs: 756019830
+frame_id: '' and grasped_object_mask header seq: 194612
+stamp:
+  secs: 1735699181
+  nsecs: 518287182
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699182.010400]: received mask_obs with header seq: 2750
+stamp:
+  secs: 1735699182
+  nsecs:  13211965
+frame_id: '' and grasped_object_mask header seq: 194620
+stamp:
+  secs: 1735699181
+  nsecs: 784739971
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699182.273450]: received mask_obs with header seq: 2751
+stamp:
+  secs: 1735699182
+  nsecs: 276359319
+frame_id: '' and grasped_object_mask header seq: 194628
+stamp:
+  secs: 1735699182
+  nsecs:  51183462
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699182.528837]: received mask_obs with header seq: 2752
+stamp:
+  secs: 1735699182
+  nsecs: 532434225
+frame_id: '' and grasped_object_mask header seq: 194635
+stamp:
+  secs: 1735699182
+  nsecs: 284319401
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699182.785968]: received mask_obs with header seq: 2753
+stamp:
+  secs: 1735699182
+  nsecs: 789589166
+frame_id: '' and grasped_object_mask header seq: 194643
+stamp:
+  secs: 1735699182
+  nsecs: 550761223
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699183.046076]: received mask_obs with header seq: 2754
+stamp:
+  secs: 1735699183
+  nsecs:  49440622
+frame_id: '' and grasped_object_mask header seq: 194651
+stamp:
+  secs: 1735699182
+  nsecs: 817206144
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699183.311548]: received mask_obs with header seq: 2755
+stamp:
+  secs: 1735699183
+  nsecs: 315151929
+frame_id: '' and grasped_object_mask header seq: 194659
+stamp:
+  secs: 1735699183
+  nsecs:  83658218
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699183.576370]: received mask_obs with header seq: 2756
+stamp:
+  secs: 1735699183
+  nsecs: 580031156
+frame_id: '' and grasped_object_mask header seq: 194667
+stamp:
+  secs: 1735699183
+  nsecs: 350110292
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699183.838235]: received mask_obs with header seq: 2757
+stamp:
+  secs: 1735699183
+  nsecs: 841592550
+frame_id: '' and grasped_object_mask header seq: 194675
+stamp:
+  secs: 1735699183
+  nsecs: 616562605
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699184.094202]: received mask_obs with header seq: 2758
+stamp:
+  secs: 1735699184
+  nsecs:  97671508
+frame_id: '' and grasped_object_mask header seq: 194682
+stamp:
+  secs: 1735699183
+  nsecs: 849708557
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is 103921619 for nearest: 1735699184220082489 - target: 1735699184116160870 at idx 249
+[INFO] [1735699184.371372]: received mask_obs with header seq: 2759
+stamp:
+  secs: 1735699184
+  nsecs: 374807119
+frame_id: '' and grasped_object_mask header seq: 194690
+stamp:
+  secs: 1735699184
+  nsecs: 116160870
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699184.642030]: received mask_obs with header seq: 2760
+stamp:
+  secs: 1735699184
+  nsecs: 645528078
+frame_id: '' and grasped_object_mask header seq: 194698
+stamp:
+  secs: 1735699184
+  nsecs: 382614613
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699184.897791]: received mask_obs with header seq: 2761
+stamp:
+  secs: 1735699184
+  nsecs: 900455713
+frame_id: '' and grasped_object_mask header seq: 194706
+stamp:
+  secs: 1735699184
+  nsecs: 649067163
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699185.004420]: Storing episode...
+[rosbag_record-15] killing on exit
+[print_text-14] killing on exit
+[pub_text-13] killing on exit
+[INFO] [1735699185.152008]: received mask_obs with header seq: 2762
+stamp:
+  secs: 1735699185
+  nsecs: 154762983
+frame_id: '' and grasped_object_mask header seq: 194714
+stamp:
+  secs: 1735699184
+  nsecs: 915515423
+frame_id: "camera_color_optical_frame"
+[INFO] [1735699185.797092]: Stored episode 5.
+[INFO] [1735699185.805713]: Saving video...
+[INFO] [1735699186.258454]: Video saved!
+[INFO] [1735699186.357373]: resetting environment
+[INFO] [1735699186.410132]: cleared current plan
+[INFO] [1735699186.413165]: moving to home
+[WARN] [1735699186.346441]: Plan exhausted
+[WARN] [1735699186.384482]: Plan exhausted
+[INFO] [1735699190.027950]: reached home
+[INFO] [1735699190.030660]: reset action history
+[INFO] [1735699192.833293]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/requirements.txt b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-metadata.json b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..fc05decfc26e8eb9c310f21e5edd588c4fda2925
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-01T02:35:03.799114",
+    "startedAt": "2025-01-01T02:35:03.434313",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "efd33a493dbad5ba8b9c770955ac9f0942bd971a"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3749.55275,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.978,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 515.321044921875
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..273bf80a96838b219ccaefaa9d47269d60a3b524
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "c8a408e3df7bfc74cc89f4bb1e1a72dba36827787ddaf45bcdc22e92b20b160b", "size": 1465413, "path": "media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4"}, "global_step": 536, "_timestamp": 1735699205.7070858, "_runtime": 302.26486682891846, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "eb902dfcfac85de572711b69b5169f14cd46b193216b8d0fb49f7108c7df956a", "size": 609556, "path": "media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4"}, "eval/num_success": 3.0, "episode": 10.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.1285562515258789, "eval/success_rate_ub_0.01": 0.9510095119476318, "eval/success_rate_lb_0.025": 0.14756186306476593, "eval/success_rate_ub_0.025": 0.9175734519958496, "eval/success_rate_lb_0.05": 0.25050920248031616, "eval/success_rate_ub_0.05": 0.882312536239624, "eval/success_rate_lb_0.1": 0.3999994695186615, "eval/success_rate_ub_0.1": 0.8205093145370483, "eval/2_eval": {"_type": "video-file", "sha256": "dea02de20bdd586cd5fdc45e12e91aa0201c0f251b47214c0c207f72e26d4bbf", "size": 1059657, "path": "media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "d10ed98fe5d597fa5a89e7f68b60f25182c82904b1d424cc5d6328ea83717bda", "size": 1466419, "path": "media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "578f39c6eeca4936625ce4639b6f651b2683d682a9c827e509298a184631d901", "size": 1516972, "path": "media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "bb2b66ada2f22e77fd7f6070fb35ca74f8550a1117f4b1b40bac2837d171cf7c", "size": 400, "artifact_path": "wandb-client-artifact://3cg9v883nu621a19m0l1abpa3wovo5bm9rw64c3fg2oq47q0iphj96ilqaqsnp6wp02ypanyk9gi30ee5w1vdm67dgnwlxrm0spian10ukof7coh54zbaytd3emq9nqs/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ehfokyi41oyj19gai1knmmoci3tob54s02g0ijf1fk4n4zr7f88wtcg3vkbc3tbhvna72igtarb60pk7v4klcck6apxnqtkp7k03r97d6kywkl3rl09chwjk7unsykv5:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 5, "total_num_episodes": 10, "total_success_rate": 0.5, "total_success_rate_lb_0.01": 0.16378831870127553, "total_success_rate_ub_0.01": 0.8324364423086958, "total_success_rate_lb_0.025": 0.22730267053336026, "total_success_rate_ub_0.025": 0.8105924724911346, "total_success_rate_lb_0.05": 0.2652893662922119, "total_success_rate_ub_0.05": 0.7343519180544403, "total_success_rate_lb_0.1": 0.279645524963104, "total_success_rate_ub_0.1": 0.7253974377658183, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "03024be658973921c64aeaa5fbee75268142c29ef013e08ad3cae7d084f40d3d", "size": 505, "artifact_path": "wandb-client-artifact://smkcb1hb8y8f7ab70o573mrdrfb7cmqi1totgv50ggew621gleolltnqswao0989ngc1y6nz20g3dystvhtdyv6ghbsh1aotrpdx5jm3zckountpznxi0ybspqd8p3ra/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zd3wyeppdiiadxchny5bv3m6poin7wxfy7ueutrmqlth8rvzuh131j87mg50sir1vlxxboamgkgshjd9ekgq8mw2kqp7jhwv963unbduq6bp9kptp43gzury7jj21plm:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 305}, "eval/total_success_rate": 0.5, "eval/total_success_rate_lb_0.01": 0.1637883186340332, "eval/total_success_rate_ub_0.01": 0.8324364423751831, "eval/total_success_rate_lb_0.025": 0.2273026704788208, "eval/total_success_rate_ub_0.025": 0.8105924725532532, "eval/total_success_rate_lb_0.05": 0.2652893662452698, "eval/total_success_rate_ub_0.05": 0.7343519330024719, "eval/total_success_rate_lb_0.1": 0.27964553236961365, "eval/total_success_rate_ub_0.1": 0.7253974080085754}
\ No newline at end of file
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/logs/debug-internal.log b/c7zbixne/wandb/run-20241231_213503-c7zbixne/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..3d270f1304aa0a4788b0653ac6b57efc21702435
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/logs/debug-internal.log
@@ -0,0 +1,883 @@
+2024-12-31 21:35:03,442 INFO    StreamThr :1748203 [internal.py:wandb_internal():85] W&B internal server running at pid: 1748203, started at: 2024-12-31 21:35:03.441695
+2024-12-31 21:35:03,443 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status
+2024-12-31 21:35:03,453 INFO    WriterThread:1748203 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/run-c7zbixne.wandb
+2024-12-31 21:35:03,454 DEBUG   SenderThread:1748203 [sender.py:send():379] send: header
+2024-12-31 21:35:03,464 DEBUG   SenderThread:1748203 [sender.py:send():379] send: run
+2024-12-31 21:35:03,684 INFO    SenderThread:1748203 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files
+2024-12-31 21:35:03,684 INFO    SenderThread:1748203 [sender.py:_start_run_threads():1188] run started: c7zbixne with start time 1735698903.442219
+2024-12-31 21:35:03,701 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: check_version
+2024-12-31 21:35:03,701 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: check_version
+2024-12-31 21:35:03,776 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: run_start
+2024-12-31 21:35:03,785 DEBUG   HandlerThread:1748203 [system_info.py:__init__():26] System info init
+2024-12-31 21:35:03,785 DEBUG   HandlerThread:1748203 [system_info.py:__init__():41] System info init done
+2024-12-31 21:35:03,785 INFO    HandlerThread:1748203 [system_monitor.py:start():194] Starting system monitor
+2024-12-31 21:35:03,785 INFO    SystemMonitor:1748203 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-31 21:35:03,785 INFO    HandlerThread:1748203 [system_monitor.py:probe():214] Collecting system info
+2024-12-31 21:35:03,785 INFO    SystemMonitor:1748203 [interfaces.py:start():188] Started cpu monitoring
+2024-12-31 21:35:03,786 INFO    SystemMonitor:1748203 [interfaces.py:start():188] Started disk monitoring
+2024-12-31 21:35:03,787 INFO    SystemMonitor:1748203 [interfaces.py:start():188] Started gpu monitoring
+2024-12-31 21:35:03,788 INFO    SystemMonitor:1748203 [interfaces.py:start():188] Started memory monitoring
+2024-12-31 21:35:03,788 INFO    SystemMonitor:1748203 [interfaces.py:start():188] Started network monitoring
+2024-12-31 21:35:03,799 DEBUG   HandlerThread:1748203 [system_info.py:probe():152] Probing system
+2024-12-31 21:35:03,800 DEBUG   HandlerThread:1748203 [system_info.py:_probe_git():137] Probing git
+2024-12-31 21:35:03,805 DEBUG   HandlerThread:1748203 [system_info.py:_probe_git():145] Probing git done
+2024-12-31 21:35:03,805 DEBUG   HandlerThread:1748203 [system_info.py:probe():200] Probing system done
+2024-12-31 21:35:03,805 DEBUG   HandlerThread:1748203 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T02:35:03.799114', 'startedAt': '2025-01-01T02:35:03.434313', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.55275, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.978, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 515.321044921875}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-31 21:35:03,805 INFO    HandlerThread:1748203 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-31 21:35:03,805 INFO    HandlerThread:1748203 [system_monitor.py:probe():227] Publishing system info
+2024-12-31 21:35:03,805 DEBUG   HandlerThread:1748203 [system_info.py:_save_code():44] Saving code
+2024-12-31 21:35:03,806 DEBUG   HandlerThread:1748203 [system_info.py:_save_code():67] Saving code done
+2024-12-31 21:35:03,806 DEBUG   HandlerThread:1748203 [system_info.py:_save_patches():84] Saving git patches
+2024-12-31 21:35:03,868 DEBUG   HandlerThread:1748203 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-31 21:35:03,869 INFO    HandlerThread:1748203 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-31 21:35:03,872 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:35:03,872 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-31 21:35:03,872 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-31 21:35:03,872 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-31 21:35:04,121 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-31 21:35:04,121 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: python_packages
+2024-12-31 21:35:04,121 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:35:04,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:35:04,124 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/fcqxte4d-diff.patch
+2024-12-31 21:35:04,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:04,125 INFO    HandlerThread:1748203 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/tb"
+  save: true
+}
+control {
+  relay_id: "0766f48012f149d2be96e80bee70b554"
+}
+_info {
+  stream_id: "c7zbixne"
+}
+
+2024-12-31 21:35:04,126 DEBUG   HandlerThread:1748203 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-31 21:35:04,159 INFO    wandb-upload_0:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/xj2oosdr-wandb-metadata.json
+2024-12-31 21:35:04,187 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/qe78rsai-code/FISH/eval_robot.py
+2024-12-31 21:35:04,284 DEBUG   SenderThread:1748203 [sender.py:send():379] send: telemetry
+2024-12-31 21:35:04,284 DEBUG   SenderThread:1748203 [sender.py:send():379] send: tbrecord
+2024-12-31 21:35:04,285 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:35:04,285 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0 with policy live
+2024-12-31 21:35:04,519 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/jafedgkg-events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:35:04,685 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-metadata.json
+2024-12-31 21:35:04,685 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/diff.patch
+2024-12-31 21:35:04,685 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code/FISH/eval_robot.py
+2024-12-31 21:35:04,686 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:35:04,686 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/requirements.txt
+2024-12-31 21:35:04,686 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:04,686 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code/FISH
+2024-12-31 21:35:04,686 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code
+2024-12-31 21:35:05,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:06,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:06,684 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:07,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:08,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:09,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:09,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:10,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:11,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:12,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:13,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:14,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:14,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:15,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:16,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:17,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:18,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:18,759 DEBUG   SenderThread:1748203 [sender.py:send():379] send: config
+2024-12-31 21:35:19,121 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:35:19,121 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:35:19,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:19,274 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:20,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:20,689 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:21,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:22,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:22,690 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:23,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:24,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:24,690 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:25,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:25,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:26,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:27,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:28,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:28,691 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:29,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:30,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:30,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:31,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:32,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:32,692 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:33,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:34,121 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:35:34,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:35:34,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:35,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:35,331 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:35,693 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml
+2024-12-31 21:35:36,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:36,694 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:37,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:38,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:38,694 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:39,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:40,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:40,451 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:40,695 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:41,127 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:42,127 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:42,695 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:43,127 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:44,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:44,696 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:45,127 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:45,520 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:46,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:46,697 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:47,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:48,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:48,697 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:49,121 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:35:49,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:35:49,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:50,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:50,639 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:50,698 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:51,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:52,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:52,698 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:53,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:54,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:54,699 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:55,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:55,841 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:35:56,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:56,700 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:57,129 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:58,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:35:58,700 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:35:59,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:00,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:00,701 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:01,117 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:01,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:02,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:02,701 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:03,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:03,788 DEBUG   SystemMonitor:1748203 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-31 21:36:03,789 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:36:04,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:36:04,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:36:04,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:04,702 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:05,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:06,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:06,354 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:06,702 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:07,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:08,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:08,703 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:09,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:10,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:10,372 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:36:10,372 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4 with policy now
+2024-12-31 21:36:10,372 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:36:10,373 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:36:10,373 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:36:10,374 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:36:10,588 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/ii2pmaby-media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media
+2024-12-31 21:36:10,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos
+2024-12-31 21:36:10,961 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/t9ibgwop-media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4
+2024-12-31 21:36:11,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:11,389 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:12,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:12,704 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:13,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:14,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:15,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:16,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:16,705 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:16,706 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:17,132 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:18,132 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:18,706 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:19,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:36:19,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:36:19,165 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:20,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:21,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:22,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:22,206 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:23,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:24,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:25,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:26,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:27,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:28,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:28,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:29,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:30,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:31,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:32,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:32,711 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:36:32,947 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/6vmh7e8o-events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:36:33,134 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:33,789 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:36:33,790 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:34,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:36:34,122 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:36:34,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:34,712 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:35,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:36,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:36,712 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:37,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:38,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:38,713 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:38,987 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:39,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:40,135 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:40,714 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:41,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:42,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:42,715 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:43,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:44,124 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:44,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:44,716 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:45,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:46,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:46,716 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:47,136 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:48,137 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:48,717 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:49,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:36:49,126 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:36:49,137 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:49,248 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:49,970 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:36:49,970 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4 with policy now
+2024-12-31 21:36:49,972 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:36:49,973 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:36:49,973 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:36:49,973 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:36:50,137 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:50,222 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/8z93u7al-media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4
+2024-12-31 21:36:50,718 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:50,718 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:36:50,718 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4
+2024-12-31 21:36:50,718 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval
+2024-12-31 21:36:50,978 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/8lbc22xp-media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4
+2024-12-31 21:36:51,141 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:52,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:52,718 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:53,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:54,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:54,719 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:55,127 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:36:55,145 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:56,138 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:57,139 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:58,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:36:58,720 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:36:59,145 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:00,145 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:01,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:01,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:02,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:03,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:03,790 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:37:04,122 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:37:04,123 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:37:04,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:05,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:06,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:06,236 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:07,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:08,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:09,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:10,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:11,143 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:12,126 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:12,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:13,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:14,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:15,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:16,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:17,142 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:17,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:17,578 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:37:17,578 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:37:17,579 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:37:17,727 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:37:17,727 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:37:18,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:18,727 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:19,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:37:19,123 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:37:19,165 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:20,145 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:20,728 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:21,145 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:22,150 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:22,302 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:22,728 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:23,149 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:24,150 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:24,729 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:25,151 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:26,151 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:26,730 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:27,151 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:27,478 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:28,151 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:28,731 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:29,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:30,151 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:30,731 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:31,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:32,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:32,655 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:32,732 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:33,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:33,790 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:37:34,123 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:37:34,125 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:37:34,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:34,733 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:35,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:36,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:36,734 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:37,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:37,856 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:38,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:38,734 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:39,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:40,152 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:40,735 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:41,153 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:42,154 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:42,736 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:43,078 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:43,153 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:44,177 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:44,737 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:45,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:45,908 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:37:45,908 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4 with policy now
+2024-12-31 21:37:45,913 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:37:45,913 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:37:45,913 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:37:45,914 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:37:46,119 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/ju9rtktg-media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4
+2024-12-31 21:37:46,153 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:46,737 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:46,738 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:37:46,738 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4
+2024-12-31 21:37:46,738 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval
+2024-12-31 21:37:47,001 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/fvizvw0x-media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4
+2024-12-31 21:37:47,158 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:48,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:48,738 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:48,945 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:49,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:37:49,125 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:37:49,165 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:50,157 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:50,739 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:51,157 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:52,156 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:52,739 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:37:53,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:54,130 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:54,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:55,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:56,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:57,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:58,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:37:59,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:37:59,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:00,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:01,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:02,162 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:03,163 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:03,791 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:38:04,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:38:04,125 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:38:04,161 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:04,290 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:04,414 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:38:04,414 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:38:04,414 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:38:04,743 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:38:04,744 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:04,744 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:38:04,991 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/abujme1x-events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:38:05,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:06,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:06,744 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:07,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:08,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:08,745 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:09,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:09,336 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:10,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:10,745 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:11,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:12,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:12,746 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:13,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:14,166 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:14,585 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:14,747 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:15,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:16,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:16,748 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:17,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:18,167 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:18,748 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:19,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:38:19,125 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:38:19,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:19,759 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:20,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:20,749 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:21,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:22,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:22,750 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:23,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:24,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:24,752 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:25,004 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:25,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:26,168 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:26,752 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:27,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:28,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:28,753 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:29,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:30,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:30,210 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:30,754 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:31,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:32,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:32,755 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:33,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:33,791 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:38:34,125 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:38:34,125 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:38:34,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:34,756 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:35,169 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:35,460 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:36,170 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:36,756 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:37,170 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:38,170 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:38,757 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:39,170 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:40,170 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:40,684 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:40,758 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:41,171 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:42,171 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:42,758 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:43,175 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:43,962 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:38:43,962 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4 with policy now
+2024-12-31 21:38:43,970 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:38:43,971 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:38:43,971 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:38:43,978 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:38:44,172 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:44,184 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/2up8vcgt-media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4
+2024-12-31 21:38:44,759 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:44,759 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:38:44,759 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4
+2024-12-31 21:38:44,760 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval
+2024-12-31 21:38:45,051 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/f95qu0cn-media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4
+2024-12-31 21:38:45,177 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:46,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:46,182 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:47,187 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:48,179 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:48,761 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:49,132 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:38:49,132 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:38:49,188 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:50,183 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:51,183 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:51,465 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:52,193 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:52,762 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:38:53,185 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:54,194 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:55,192 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:56,193 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:57,139 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:38:57,187 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:58,195 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:38:59,199 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:00,195 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:01,200 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:02,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:03,138 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:03,196 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:03,792 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:39:04,133 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:39:04,133 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:39:04,195 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:05,201 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:06,197 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:06,487 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:39:06,487 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:39:06,487 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:39:06,767 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:39:06,767 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:06,767 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:39:07,019 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/4bycnqb2-events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:39:07,202 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:08,165 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:08,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:08,767 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:09,202 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:10,202 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:10,768 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:11,202 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:12,203 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:12,769 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:13,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:13,337 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:14,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:14,770 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:15,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:16,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:16,770 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:17,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:18,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:18,526 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:18,771 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:19,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:39:19,128 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:39:19,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:20,204 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:20,772 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:21,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:22,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:22,773 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:23,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:23,528 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:24,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:24,773 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:25,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:26,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:26,774 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:27,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:28,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:28,722 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:28,775 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:29,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:30,205 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:30,776 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:31,206 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:32,206 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:32,776 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:33,207 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:33,792 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:39:33,792 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:34,128 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:39:34,128 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:39:34,207 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:34,777 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:35,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:36,207 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:36,778 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:37,207 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:38,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:38,779 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:38,897 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:39,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:40,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:40,779 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:41,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:42,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:42,780 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:43,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:44,094 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:44,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:44,781 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:45,208 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:46,217 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:46,326 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:39:46,326 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4 with policy now
+2024-12-31 21:39:46,345 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:39:46,346 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:39:46,346 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:39:46,364 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:39:46,546 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/gxxmyc6m-media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4
+2024-12-31 21:39:46,782 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:46,782 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:39:46,782 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4
+2024-12-31 21:39:46,782 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval
+2024-12-31 21:39:47,046 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/wr4ci5gj-media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4
+2024-12-31 21:39:47,239 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:48,246 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:48,782 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:49,132 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:39:49,132 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:39:49,234 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:49,287 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:50,237 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:50,783 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:51,241 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:52,241 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:53,241 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:54,243 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:54,785 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:39:55,148 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:39:55,270 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:56,253 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:57,250 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:58,252 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:39:59,257 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:00,251 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:01,144 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:40:01,260 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:02,252 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:03,262 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:03,792 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:40:04,131 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-31 21:40:04,132 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: stop_status
+2024-12-31 21:40:04,259 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:04,788 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:40:05,165 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 21:40:05,166 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 21:40:05,262 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:05,431 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:40:05,626 INFO    wandb-upload_2:1748203 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8715f46y
+2024-12-31 21:40:05,784 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: log_artifact
+2024-12-31 21:40:05,788 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:40:05,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json
+2024-12-31 21:40:05,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json
+2024-12-31 21:40:05,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media
+2024-12-31 21:40:05,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval
+2024-12-31 21:40:05,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table
+2024-12-31 21:40:05,795 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-31 21:40:06,026 INFO    SenderThread:1748203 [sender.py:send_request_log_artifact():1525] logged artifact run-c7zbixne-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMzI0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2OTAxMg==', 'latestArtifact': None}}
+2024-12-31 21:40:06,026 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:40:06,026 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json with policy now
+2024-12-31 21:40:06,027 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:40:06,028 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,029 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,029 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:40:06,029 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,030 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,030 DEBUG   SenderThread:1748203 [sender.py:send():379] send: telemetry
+2024-12-31 21:40:06,031 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,031 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,032 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,032 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,033 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,033 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,034 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,034 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,034 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,035 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,035 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,035 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,035 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,036 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,036 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,036 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,036 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,037 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,037 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,037 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,037 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,038 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:06,038 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: log_artifact
+2024-12-31 21:40:06,219 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/m7nfgzio-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json
+2024-12-31 21:40:06,243 INFO    wandb-upload_0:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/mu41qi4m-media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json
+2024-12-31 21:40:06,255 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:06,459 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/hh7yudnl-media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json
+2024-12-31 21:40:06,506 INFO    wandb-upload_1:1748203 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpv5ousy4x
+2024-12-31 21:40:06,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:40:06,874 INFO    SenderThread:1748203 [sender.py:send_request_log_artifact():1525] logged artifact run-c7zbixne-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMzI2Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE2OTAxNw==', 'latestArtifact': None}}
+2024-12-31 21:40:06,874 DEBUG   SenderThread:1748203 [sender.py:send():379] send: files
+2024-12-31 21:40:06,874 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json with policy now
+2024-12-31 21:40:06,875 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:40:06,875 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:06,875 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:40:06,876 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:07,117 INFO    wandb-upload_0:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/cy3wmssk-media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json
+2024-12-31 21:40:07,255 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:07,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:40:07,789 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:40:08,255 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:09,255 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:09,501 DEBUG   SenderThread:1748203 [sender.py:send():379] send: exit
+2024-12-31 21:40:09,501 INFO    SenderThread:1748203 [sender.py:send_exit():586] handling exit code: 0
+2024-12-31 21:40:09,502 INFO    SenderThread:1748203 [sender.py:send_exit():588] handling runtime: 305
+2024-12-31 21:40:09,502 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:09,502 INFO    SenderThread:1748203 [sender.py:send_exit():594] send defer
+2024-12-31 21:40:09,502 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:09,502 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-31 21:40:09,502 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:09,502 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-31 21:40:09,502 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 1
+2024-12-31 21:40:09,502 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:09,503 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-31 21:40:09,503 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:09,503 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-31 21:40:09,503 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 2
+2024-12-31 21:40:09,503 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:09,503 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-31 21:40:09,503 INFO    HandlerThread:1748203 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-31 21:40:09,503 DEBUG   SystemMonitor:1748203 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-31 21:40:09,503 DEBUG   SystemMonitor:1748203 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-31 21:40:09,504 INFO    HandlerThread:1748203 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-31 21:40:09,504 INFO    HandlerThread:1748203 [interfaces.py:finish():200] Joined disk monitor
+2024-12-31 21:40:09,556 INFO    HandlerThread:1748203 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-31 21:40:09,556 INFO    HandlerThread:1748203 [interfaces.py:finish():200] Joined memory monitor
+2024-12-31 21:40:09,556 INFO    HandlerThread:1748203 [interfaces.py:finish():200] Joined network monitor
+2024-12-31 21:40:09,557 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:09,557 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-31 21:40:09,557 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 3
+2024-12-31 21:40:09,557 DEBUG   SenderThread:1748203 [sender.py:send():379] send: stats
+2024-12-31 21:40:09,557 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:09,557 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-31 21:40:09,557 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:09,557 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-31 21:40:09,557 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 4
+2024-12-31 21:40:09,557 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:09,557 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-31 21:40:09,790 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:40:11,791 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:40:15,561 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:15,561 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:40:15,561 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-31 21:40:15,562 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:15,562 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-31 21:40:15,562 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 5
+2024-12-31 21:40:15,563 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:15,563 DEBUG   SenderThread:1748203 [sender.py:send():379] send: history
+2024-12-31 21:40:15,563 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: summary_record
+2024-12-31 21:40:15,563 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:15,563 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-31 21:40:15,564 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:15,564 DEBUG   SenderThread:1748203 [sender.py:send():379] send: summary
+2024-12-31 21:40:15,568 INFO    SenderThread:1748203 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-31 21:40:15,568 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:15,568 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-31 21:40:15,568 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 6
+2024-12-31 21:40:15,568 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:15,568 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-31 21:40:15,568 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:15,568 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-31 21:40:15,572 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: status_report
+2024-12-31 21:40:15,824 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 7
+2024-12-31 21:40:15,824 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:15,825 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-31 21:40:15,825 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:15,825 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-31 21:40:15,825 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:40:16,502 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:16,825 INFO    Thread-12 :1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml
+2024-12-31 21:40:18,018 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 8
+2024-12-31 21:40:18,019 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:18,019 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:18,019 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-31 21:40:18,019 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:18,019 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-31 21:40:18,019 INFO    SenderThread:1748203 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-31 21:40:18,021 INFO    SenderThread:1748203 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-31 21:40:18,023 INFO    SenderThread:1748203 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-31 21:40:18,031 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 9
+2024-12-31 21:40:18,031 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:18,032 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-31 21:40:18,032 DEBUG   SenderThread:1748203 [sender.py:send():379] send: artifact
+2024-12-31 21:40:18,448 INFO    SenderThread:1748203 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDIwNA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDEzMDIwNA==', 'versionIndex': 393}}}
+2024-12-31 21:40:18,448 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:18,448 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-31 21:40:18,448 INFO    SenderThread:1748203 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-31 21:40:18,503 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml config.yaml
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0 events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json wandb-summary.json
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/requirements.txt requirements.txt
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-metadata.json wandb-metadata.json
+2024-12-31 21:40:18,827 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log output.log
+2024-12-31 21:40:18,828 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/diff.patch diff.patch
+2024-12-31 21:40:18,828 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-31 21:40:18,829 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4 media/videos/eval/4_eval_4_578f39c6eeca4936625c.mp4
+2024-12-31 21:40:18,830 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4 media/videos/eval/1_eval_1_eb902dfcfac85de57271.mp4
+2024-12-31 21:40:18,830 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4 media/videos/eval/3_eval_3_d10ed98fe5d597fa5a89.mp4
+2024-12-31 21:40:18,830 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4 media/videos/eval/2_eval_2_dea02de20bdd586cd5fd.mp4
+2024-12-31 21:40:18,830 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4 media/videos/eval/0_eval_0_c8a408e3df7bfc74cc89.mp4
+2024-12-31 21:40:18,834 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json media/table/eval/total_success_rate_ci_6_03024be658973921c64a.table.json
+2024-12-31 21:40:18,835 INFO    SenderThread:1748203 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json media/table/eval/success_rate_ci_5_bb2b66ada2f22e77fd7f.table.json
+2024-12-31 21:40:18,835 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 10
+2024-12-31 21:40:18,835 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:18,835 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:18,835 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-31 21:40:18,837 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:18,837 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-31 21:40:18,837 INFO    SenderThread:1748203 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 21:40:19,007 INFO    wandb-upload_0:1748203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/wandb-summary.json
+2024-12-31 21:40:19,008 INFO    wandb-upload_1:1748203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/config.yaml
+2024-12-31 21:40:19,064 INFO    wandb-upload_2:1748203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/requirements.txt
+2024-12-31 21:40:19,074 INFO    wandb-upload_4:1748203 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/files/output.log
+2024-12-31 21:40:19,110 INFO    wandb-upload_3:1748203 [upload_job.py:push():130] Uploaded file /tmp/tmp2ssyz0liwandb/edl5p080-events.out.tfevents.1735698904.leonmkim-ROG-Strix-G15CS-G15CS.1748029.0
+2024-12-31 21:40:19,310 INFO    Thread-11 (_thread_body):1748203 [sender.py:transition_state():614] send defer: 11
+2024-12-31 21:40:19,311 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:19,311 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-31 21:40:19,311 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:19,311 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-31 21:40:19,311 INFO    SenderThread:1748203 [file_pusher.py:join():175] waiting for file pusher
+2024-12-31 21:40:19,314 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 12
+2024-12-31 21:40:19,314 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:19,314 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-31 21:40:19,314 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:19,314 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-31 21:40:19,315 INFO    SenderThread:1748203 [file_stream.py:finish():601] file stream finish called
+2024-12-31 21:40:19,504 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:19,533 INFO    SenderThread:1748203 [file_stream.py:finish():605] file stream finish is done
+2024-12-31 21:40:19,533 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 13
+2024-12-31 21:40:19,533 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:19,533 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:19,533 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-31 21:40:19,533 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:19,533 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-31 21:40:19,534 INFO    SenderThread:1748203 [sender.py:transition_state():614] send defer: 14
+2024-12-31 21:40:19,534 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: defer
+2024-12-31 21:40:19,534 INFO    HandlerThread:1748203 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-31 21:40:19,534 DEBUG   SenderThread:1748203 [sender.py:send():379] send: final
+2024-12-31 21:40:19,534 DEBUG   SenderThread:1748203 [sender.py:send():379] send: footer
+2024-12-31 21:40:19,535 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: defer
+2024-12-31 21:40:19,535 INFO    SenderThread:1748203 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-31 21:40:19,535 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:19,535 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: server_info
+2024-12-31 21:40:19,535 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-31 21:40:19,535 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-31 21:40:19,536 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-31 21:40:19,536 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-31 21:40:19,537 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:19,537 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: server_info
+2024-12-31 21:40:19,603 DEBUG   SenderThread:1748203 [sender.py:send_request():406] send_request: poll_exit
+2024-12-31 21:40:19,604 INFO    MainThread:1748203 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-31 21:40:19,604 INFO    MainThread:1748203 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-31 21:40:19,604 INFO    MainThread:1748203 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-31 21:40:19,604 DEBUG   HandlerThread:1748203 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-31 21:40:19,605 INFO    HandlerThread:1748203 [handler.py:finish():882] shutting down handler
+2024-12-31 21:40:20,537 INFO    WriterThread:1748203 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/run-c7zbixne.wandb
+2024-12-31 21:40:20,604 INFO    SenderThread:1748203 [sender.py:finish():1615] shutting down sender
+2024-12-31 21:40:20,604 INFO    SenderThread:1748203 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-31 21:40:20,604 INFO    SenderThread:1748203 [file_pusher.py:join():175] waiting for file pusher
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/logs/debug.log b/c7zbixne/wandb/run-20241231_213503-c7zbixne/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..f3238057c2f75457e753c19e4fc0cbdec21b5efd
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/logs/debug.log
@@ -0,0 +1,31 @@
+2024-12-31 21:35:03,436 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Configure stats pid to 1748029
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/settings
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/logs/debug.log
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/wandb/run-20241231_213503-c7zbixne/logs/debug-internal.log
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:init():569] calling init triggers
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1006_0', 'wandb_notes': '1006_0_req_1352_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne'}
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:init():619] starting backend
+2024-12-31 21:35:03,437 INFO    MainThread:1748029 [wandb_init.py:init():623] setting up manager
+2024-12-31 21:35:03,441 INFO    MainThread:1748029 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-31 21:35:03,441 INFO    MainThread:1748029 [wandb_init.py:init():631] backend started and connected
+2024-12-31 21:35:03,453 INFO    MainThread:1748029 [wandb_init.py:init():720] updated telemetry
+2024-12-31 21:35:03,461 INFO    MainThread:1748029 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-31 21:35:03,701 INFO    MainThread:1748029 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-31 21:35:03,770 INFO    MainThread:1748029 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-31 21:35:03,771 INFO    MainThread:1748029 [wandb_init.py:init():804] starting run threads in backend
+2024-12-31 21:35:04,121 INFO    MainThread:1748029 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-31 21:35:04,121 INFO    MainThread:1748029 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-31 21:35:04,121 INFO    MainThread:1748029 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-31 21:35:04,122 INFO    MainThread:1748029 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-31 21:35:04,123 INFO    MainThread:1748029 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-31 21:35:04,125 INFO    MainThread:1748029 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/c7zbixne/tb, True
+2024-12-31 21:35:18,758 INFO    MainThread:1748029 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2024-12-31 21:40:20,605 WARNING MsgRouterThr:1748029 [router.py:message_loop():77] message_loop has been closed
diff --git a/c7zbixne/wandb/run-20241231_213503-c7zbixne/run-c7zbixne.wandb b/c7zbixne/wandb/run-20241231_213503-c7zbixne/run-c7zbixne.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..c9a9aa18f655c1e84075bec84b933605bcc7f837
--- /dev/null
+++ b/c7zbixne/wandb/run-20241231_213503-c7zbixne/run-c7zbixne.wandb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:682802c4047b056bba91c2197b17f2551d64b4e0e291c1b3ea0d21ca72da328b
+size 260659
diff --git a/djs0twrt/.hydra/config.yaml b/djs0twrt/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90
--- /dev/null
+++ b/djs0twrt/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1006_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/djs0twrt/.hydra/hydra.yaml b/djs0twrt/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..95999915080cb54262d33c5e60c6edc36af09a18
--- /dev/null
+++ b/djs0twrt/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/djs0twrt
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/djs0twrt/.hydra/overrides.yaml b/djs0twrt/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/djs0twrt/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/djs0twrt/eval_policy.log b/djs0twrt/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..6d552d395f7cf8daaae2a88da05c4b4495e3dae9
--- /dev/null
+++ b/djs0twrt/eval_policy.log
@@ -0,0 +1,15 @@
+[2024-12-31 23:32:17,512][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-31 23:32:17,515][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-31 23:32:21,725][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+
diff --git a/f8lc8kux/.hydra/config.yaml b/f8lc8kux/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..05485f98b3c08bece875a460fe25a0cce7504f90
--- /dev/null
+++ b/f8lc8kux/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1006_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/f8lc8kux/.hydra/hydra.yaml b/f8lc8kux/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4453a1ef12a24a068eec2906aba234cb1b980027
--- /dev/null
+++ b/f8lc8kux/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1006_0/f8lc8kux
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/f8lc8kux/.hydra/overrides.yaml b/f8lc8kux/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/f8lc8kux/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/f8lc8kux/eval_policy.log b/f8lc8kux/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..822fb4313ab1e880839bc6880af2f0c3fb251813
--- /dev/null
+++ b/f8lc8kux/eval_policy.log
@@ -0,0 +1,15 @@
+[2025-01-01 11:02:53,930][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-01 11:02:53,934][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-01 11:02:58,165][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+
diff --git a/lydwde6h/episode_rosbags/episode_2_2025-01-01-11-18-39.bag b/lydwde6h/episode_rosbags/episode_2_2025-01-01-11-18-39.bag
new file mode 100644
index 0000000000000000000000000000000000000000..34bf2db041cb25136d17ff4d927a549f795a3285
--- /dev/null
+++ b/lydwde6h/episode_rosbags/episode_2_2025-01-01-11-18-39.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81dfa80a32be164cdd8f3e2c97d34d006cd8a354e63dff77514334837521ad23
+size 1424275404
diff --git a/lydwde6h/episode_rosbags/episode_3_2025-01-01-11-19-37.bag b/lydwde6h/episode_rosbags/episode_3_2025-01-01-11-19-37.bag
new file mode 100644
index 0000000000000000000000000000000000000000..b3ef5e713e0ea92d70e668d7f7f4c544209dcdaa
--- /dev/null
+++ b/lydwde6h/episode_rosbags/episode_3_2025-01-01-11-19-37.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba2c86facf903c9e15d0d451cf09013d4db4d3b9a42aa47007735e09f010a983
+size 1560809649
diff --git a/lydwde6h/episode_rosbags/episode_4_2025-01-01-11-20-32.bag b/lydwde6h/episode_rosbags/episode_4_2025-01-01-11-20-32.bag
new file mode 100644
index 0000000000000000000000000000000000000000..514339a2dc3e557ab3ecf6bc96a0156a343c72a5
--- /dev/null
+++ b/lydwde6h/episode_rosbags/episode_4_2025-01-01-11-20-32.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0e7e02e796234416ef8c606a69cfb1017f9a8fbe8ddf1387048fc81a025a95b
+size 1363996745