diff --git a/.gitattributes b/.gitattributes
index 72634966fe2fe7b5cbaccf26f6275812368e6dc6..4a3023c0f15ef42270d53135f632dd3d7279032b 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -473,3 +473,51 @@ bmk18y9z/episode_rosbags/episode_6_2025-01-07-22-54-12.bag filter=lfs diff=lfs m
 4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/11_eval_11_6141e16e285aea41414e.mp4 filter=lfs diff=lfs merge=lfs -text
 4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/12_eval_12_212b177b80b2034deb4f.mp4 filter=lfs diff=lfs merge=lfs -text
 4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/9_eval_9_c7576684fb563a572588.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/2_eval_2_537ec4e4ec2357120fef.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/3_eval_3_5243c77fb64415fcde38.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/4_eval_4_123736b8ecd81d086c47.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/18_eval_18_fd5b9c7ce55fe4462130.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/13_eval_13_49e329d2ff2e935251c1.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/14_eval_14_b1f7698aaf2c3d6e6abf.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/16_eval_16_a93b394f6e07a47f91ce.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/6_eval_6_718698df80de1dc3b1e7.mp4 filter=lfs diff=lfs merge=lfs -text
+c9fefav0/episode_rosbags/episode_0_2025-01-06-21-06-27.bag filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/17_eval_17_e673279ceb422c546539.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/15_eval_15_b314c0181db144f9c4bb.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/7_eval_7_0c0abd0712ae602c07c3.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/0_eval_0_ae65028e83f44b9ac5b1.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+c9fefav0/episode_rosbags/episode_4_2025-01-06-21-09-26.bag filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+3d3i07nt/episode_rosbags/episode_2_2025-01-06-21-02-35.bag filter=lfs diff=lfs merge=lfs -text
+4fmihp69/episode_rosbags/episode_3_2025-01-07-22-09-09.bag filter=lfs diff=lfs merge=lfs -text
+4fmihp69/episode_rosbags/episode_4_2025-01-07-22-09-52.bag filter=lfs diff=lfs merge=lfs -text
+bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4 filter=lfs diff=lfs merge=lfs -text
+bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4 filter=lfs diff=lfs merge=lfs -text
+bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4 filter=lfs diff=lfs merge=lfs -text
+bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4 filter=lfs diff=lfs merge=lfs -text
+bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4 filter=lfs diff=lfs merge=lfs -text
+4fmihp69/episode_rosbags/episode_2_2025-01-07-22-08-27.bag filter=lfs diff=lfs merge=lfs -text
+bkletf5j/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+bkletf5j/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+bkletf5j/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+bkletf5j/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+bkletf5j/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
diff --git a/3d3i07nt/episode_rosbags/episode_2_2025-01-06-21-02-35.bag b/3d3i07nt/episode_rosbags/episode_2_2025-01-06-21-02-35.bag
new file mode 100644
index 0000000000000000000000000000000000000000..f6c2c3c4ecdf21ab0fc550191e1e621238a2553c
--- /dev/null
+++ b/3d3i07nt/episode_rosbags/episode_2_2025-01-06-21-02-35.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db0b689d49eea41c1743f5492b49ed351384554397594bb2fb35b8906956e607
+size 1413121843
diff --git a/4fmihp69/.hydra/config.yaml b/4fmihp69/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1a56df077cebbbb54879d37edcc83ff7d1e7c8eb
--- /dev/null
+++ b/4fmihp69/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 20
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1002_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/4fmihp69/.hydra/hydra.yaml b/4fmihp69/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fc9bf4d292cbd63d8badc8e2f59fadab3cc81654
--- /dev/null
+++ b/4fmihp69/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/4fmihp69/.hydra/overrides.yaml b/4fmihp69/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/4fmihp69/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/4fmihp69/episode_rosbags/episode_2_2025-01-07-22-08-27.bag b/4fmihp69/episode_rosbags/episode_2_2025-01-07-22-08-27.bag
new file mode 100644
index 0000000000000000000000000000000000000000..17fcf2a605683f176a88c74c1ca5b8161e19deee
--- /dev/null
+++ b/4fmihp69/episode_rosbags/episode_2_2025-01-07-22-08-27.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:043761c26ac246cb92cfbae7e064fd8672f3f268a46a6105d061ca36f4b0d20c
+size 1411056075
diff --git a/4fmihp69/episode_rosbags/episode_3_2025-01-07-22-09-09.bag b/4fmihp69/episode_rosbags/episode_3_2025-01-07-22-09-09.bag
new file mode 100644
index 0000000000000000000000000000000000000000..98321539f75b13a8ea07c51337fa3b155aa3616d
--- /dev/null
+++ b/4fmihp69/episode_rosbags/episode_3_2025-01-07-22-09-09.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b24d39303ac5a858f58b6d42916ba22dbfb88407b11a9c5201329b098070c501
+size 1407832609
diff --git a/4fmihp69/episode_rosbags/episode_4_2025-01-07-22-09-52.bag b/4fmihp69/episode_rosbags/episode_4_2025-01-07-22-09-52.bag
new file mode 100644
index 0000000000000000000000000000000000000000..3a2d4abf028368ed4bbc4dce388ebe82a21d8e2c
--- /dev/null
+++ b/4fmihp69/episode_rosbags/episode_4_2025-01-07-22-09-52.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11d8c765f1dca02888e208646b35a256943ec9952c29d3b8327193fae3a9bc1a
+size 1411111409
diff --git a/4fmihp69/eval_video/0_eval.mp4 b/4fmihp69/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ba5d39fa607534530896cd2e0fd5baf8e2785bb2
--- /dev/null
+++ b/4fmihp69/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae65028e83f44b9ac5b1292e9a19a482040bef9b85dc8452c1003e498adcf200
+size 1203236
diff --git a/4fmihp69/eval_video/10_eval.mp4 b/4fmihp69/eval_video/10_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bc148893184a69f308543e84014e1977301aa0b7
--- /dev/null
+++ b/4fmihp69/eval_video/10_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e9fbf635a50e60a4cb3bc19311a779d79512d4e50a6bdec2a27735c3a018973
+size 1317315
diff --git a/4fmihp69/eval_video/11_eval.mp4 b/4fmihp69/eval_video/11_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..99b61cacce3c19c4337c57a2371db3fff28c3b18
--- /dev/null
+++ b/4fmihp69/eval_video/11_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6141e16e285aea41414e683be227f5ddaab91e58c74c8fe9a6908e9a5ab511ed
+size 1325039
diff --git a/4fmihp69/eval_video/12_eval.mp4 b/4fmihp69/eval_video/12_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4655f02574e1ff2e1f1de6829c64715e8232c627
--- /dev/null
+++ b/4fmihp69/eval_video/12_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:212b177b80b2034deb4f20821b47755bf2b9d214e66cc422750d97dfd12630bc
+size 1365265
diff --git a/4fmihp69/eval_video/13_eval.mp4 b/4fmihp69/eval_video/13_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..31e731a727efa864d75f952f2d0f302d58f21c26
--- /dev/null
+++ b/4fmihp69/eval_video/13_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49e329d2ff2e935251c138c38bd869e83a370b6a20c29a6f3f6f4fb5241a5926
+size 1214289
diff --git a/4fmihp69/eval_video/14_eval.mp4 b/4fmihp69/eval_video/14_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..e84b263bdcd49118856a5c7fcf5cf1eda91b77ae
--- /dev/null
+++ b/4fmihp69/eval_video/14_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1f7698aaf2c3d6e6abf7855ee3e636573dce1a6d4641acb7f13c8ce0412b52c
+size 1410819
diff --git a/4fmihp69/eval_video/15_eval.mp4 b/4fmihp69/eval_video/15_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bf2ba073c69b96fb43c0b49d9733c266e76ca150
--- /dev/null
+++ b/4fmihp69/eval_video/15_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b314c0181db144f9c4bbd755580f55f4fa904dbd7bf96ae9ebbdf5aa6aea7161
+size 1504211
diff --git a/4fmihp69/eval_video/16_eval.mp4 b/4fmihp69/eval_video/16_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..c86a100cf93267dc14f8a7f1f54b80316ab3e207
--- /dev/null
+++ b/4fmihp69/eval_video/16_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a93b394f6e07a47f91ce5c6436e2ac6f677aa64e46d4e4c1331cad98e6429f67
+size 1441574
diff --git a/4fmihp69/eval_video/17_eval.mp4 b/4fmihp69/eval_video/17_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..20f461948c92ec9f52801077ef1191e58098df79
--- /dev/null
+++ b/4fmihp69/eval_video/17_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e673279ceb422c54653970040c48bf551dbc8cea9639629ae5c5f872e26f2b86
+size 1438873
diff --git a/4fmihp69/eval_video/18_eval.mp4 b/4fmihp69/eval_video/18_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5be9562aa1cb6dbae4e2c76786a3a98bb51262bd
--- /dev/null
+++ b/4fmihp69/eval_video/18_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd5b9c7ce55fe446213004b70d71e620113f558086949526d140680e6e08433b
+size 1143394
diff --git a/4fmihp69/eval_video/19_eval.mp4 b/4fmihp69/eval_video/19_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..b09d9cd97bf139a37cf2a56a5811eec017d7d7b1
--- /dev/null
+++ b/4fmihp69/eval_video/19_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b857c64ab339c17adb153080a0a92442b3ece13d773e460489d0645e2a50a415
+size 1247835
diff --git a/4fmihp69/eval_video/1_eval.mp4 b/4fmihp69/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ad0e32a59b8ed4f835cd45a594b00a39c347e865
--- /dev/null
+++ b/4fmihp69/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5acd6aafe960785647d8b9e25843b0bfe650a9f5b093f48cab02c8bb3ca18dc
+size 1200266
diff --git a/4fmihp69/eval_video/2_eval.mp4 b/4fmihp69/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..3eb03850555c9c68aa1233f66338e7705a22905d
--- /dev/null
+++ b/4fmihp69/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:537ec4e4ec2357120fef5cc7f4edbf14515d4b2e6bdfab19c7eeeacbe708bc89
+size 1203819
diff --git a/4fmihp69/eval_video/3_eval.mp4 b/4fmihp69/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..d1ff9e9073565837197ce99a5f2af69d6002c56f
--- /dev/null
+++ b/4fmihp69/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5243c77fb64415fcde38b5e587dfe83a26bc73991f69876d74ed1cbc67f2c743
+size 1150313
diff --git a/4fmihp69/eval_video/4_eval.mp4 b/4fmihp69/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..d63c095794fbac5596a26e43c50fbf2414d43daa
--- /dev/null
+++ b/4fmihp69/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:123736b8ecd81d086c472194678ae8960360473b3f54800e2d9f897b23c89739
+size 1255949
diff --git a/4fmihp69/eval_video/5_eval.mp4 b/4fmihp69/eval_video/5_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1eeedf2322a05614aaf36032b6d6a5b9b062bfde
--- /dev/null
+++ b/4fmihp69/eval_video/5_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdc3dec2e2f95a3c767dc580906d7f9f320e106e08f6a4b27752d14fc60dd841
+size 1148520
diff --git a/4fmihp69/eval_video/6_eval.mp4 b/4fmihp69/eval_video/6_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5aef26bc90b4757937ad57eab628a0204e556bb7
--- /dev/null
+++ b/4fmihp69/eval_video/6_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:718698df80de1dc3b1e7aea8932c6797bc8d2289782b524986f54f2abbffc39c
+size 1163347
diff --git a/4fmihp69/eval_video/7_eval.mp4 b/4fmihp69/eval_video/7_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..fd21c9f174fb7cadb2d57fccdb1067cb7449ff3a
--- /dev/null
+++ b/4fmihp69/eval_video/7_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c0abd0712ae602c07c33b31081a76a7241f86938b9d37db8c91daabb3f3f779
+size 1201147
diff --git a/4fmihp69/eval_video/8_eval.mp4 b/4fmihp69/eval_video/8_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..be0da24dcd662534c5eae6bbc323f736c8d9a7c0
--- /dev/null
+++ b/4fmihp69/eval_video/8_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8cc872cbcaee100937a0d3546df9603678e05465b5928a259b9ed6b4d1ae545
+size 1257988
diff --git a/4fmihp69/eval_video/9_eval.mp4 b/4fmihp69/eval_video/9_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..614e57560b63efb3aff5a4bfebe5766015dcddee
--- /dev/null
+++ b/4fmihp69/eval_video/9_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7576684fb563a572588fe00177b0da5477339e45aaf0f59ca72100987f63cfa
+size 1299376
diff --git a/4fmihp69/tb/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0 b/4fmihp69/tb/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
new file mode 100644
index 0000000000000000000000000000000000000000..a73b9dce1c4f27524234b294fe0143152fc71220
--- /dev/null
+++ b/4fmihp69/tb/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca15f9bcf722c39eaf851a1c07befd8105ae64c156772bc2d5af22a4b8608749
+size 22784
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/code/FISH/eval_robot.py b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..b10d2043a20baa9d07c1baf6c0802d5f9b401012
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.3, 0.1003983021581674, 0.558511674392442, 0.1215232685965274, 0.502513885497544, 0.16696532077297443, 0.48316777497866625, 0.17897007993323794, 0.4645809233259468]]}
\ No newline at end of file
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval/total_success_rate_ci_21_54cfd2c4c0340bc54565.table.json b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval/total_success_rate_ci_21_54cfd2c4c0340bc54565.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..5ef6e45f7cfa602f2b048b0f0a3d5259b986b81d
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval/total_success_rate_ci_21_54cfd2c4c0340bc54565.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[41, 105, 0.3904761904761905, 0.286629945082423, 0.5046236664047531, 0.30276687447201045, 0.4824152737891081, 0.3192752711836344, 0.46708489955129906, 0.3268393464732785, 0.4542280621918193]]}
\ No newline at end of file
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/0_eval_0_ae65028e83f44b9ac5b1.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/0_eval_0_ae65028e83f44b9ac5b1.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ba5d39fa607534530896cd2e0fd5baf8e2785bb2
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/0_eval_0_ae65028e83f44b9ac5b1.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae65028e83f44b9ac5b1292e9a19a482040bef9b85dc8452c1003e498adcf200
+size 1203236
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/13_eval_13_49e329d2ff2e935251c1.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/13_eval_13_49e329d2ff2e935251c1.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..31e731a727efa864d75f952f2d0f302d58f21c26
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/13_eval_13_49e329d2ff2e935251c1.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49e329d2ff2e935251c138c38bd869e83a370b6a20c29a6f3f6f4fb5241a5926
+size 1214289
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/14_eval_14_b1f7698aaf2c3d6e6abf.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/14_eval_14_b1f7698aaf2c3d6e6abf.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..e84b263bdcd49118856a5c7fcf5cf1eda91b77ae
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/14_eval_14_b1f7698aaf2c3d6e6abf.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1f7698aaf2c3d6e6abf7855ee3e636573dce1a6d4641acb7f13c8ce0412b52c
+size 1410819
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/15_eval_15_b314c0181db144f9c4bb.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/15_eval_15_b314c0181db144f9c4bb.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..bf2ba073c69b96fb43c0b49d9733c266e76ca150
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/15_eval_15_b314c0181db144f9c4bb.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b314c0181db144f9c4bbd755580f55f4fa904dbd7bf96ae9ebbdf5aa6aea7161
+size 1504211
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/16_eval_16_a93b394f6e07a47f91ce.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/16_eval_16_a93b394f6e07a47f91ce.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..c86a100cf93267dc14f8a7f1f54b80316ab3e207
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/16_eval_16_a93b394f6e07a47f91ce.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a93b394f6e07a47f91ce5c6436e2ac6f677aa64e46d4e4c1331cad98e6429f67
+size 1441574
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/17_eval_17_e673279ceb422c546539.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/17_eval_17_e673279ceb422c546539.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..20f461948c92ec9f52801077ef1191e58098df79
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/17_eval_17_e673279ceb422c546539.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e673279ceb422c54653970040c48bf551dbc8cea9639629ae5c5f872e26f2b86
+size 1438873
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/18_eval_18_fd5b9c7ce55fe4462130.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/18_eval_18_fd5b9c7ce55fe4462130.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5be9562aa1cb6dbae4e2c76786a3a98bb51262bd
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/18_eval_18_fd5b9c7ce55fe4462130.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd5b9c7ce55fe446213004b70d71e620113f558086949526d140680e6e08433b
+size 1143394
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/2_eval_2_537ec4e4ec2357120fef.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/2_eval_2_537ec4e4ec2357120fef.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..3eb03850555c9c68aa1233f66338e7705a22905d
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/2_eval_2_537ec4e4ec2357120fef.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:537ec4e4ec2357120fef5cc7f4edbf14515d4b2e6bdfab19c7eeeacbe708bc89
+size 1203819
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/3_eval_3_5243c77fb64415fcde38.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/3_eval_3_5243c77fb64415fcde38.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..d1ff9e9073565837197ce99a5f2af69d6002c56f
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/3_eval_3_5243c77fb64415fcde38.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5243c77fb64415fcde38b5e587dfe83a26bc73991f69876d74ed1cbc67f2c743
+size 1150313
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/4_eval_4_123736b8ecd81d086c47.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/4_eval_4_123736b8ecd81d086c47.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..d63c095794fbac5596a26e43c50fbf2414d43daa
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/4_eval_4_123736b8ecd81d086c47.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:123736b8ecd81d086c472194678ae8960360473b3f54800e2d9f897b23c89739
+size 1255949
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/6_eval_6_718698df80de1dc3b1e7.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/6_eval_6_718698df80de1dc3b1e7.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5aef26bc90b4757937ad57eab628a0204e556bb7
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/6_eval_6_718698df80de1dc3b1e7.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:718698df80de1dc3b1e7aea8932c6797bc8d2289782b524986f54f2abbffc39c
+size 1163347
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/7_eval_7_0c0abd0712ae602c07c3.mp4 b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/7_eval_7_0c0abd0712ae602c07c3.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..fd21c9f174fb7cadb2d57fccdb1067cb7449ff3a
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/7_eval_7_0c0abd0712ae602c07c3.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c0abd0712ae602c07c33b31081a76a7241f86938b9d37db8c91daabb3f3f779
+size 1201147
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/logs/debug-internal.log b/4fmihp69/wandb/run-20250107_220610-4fmihp69/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..2476e314e74fc8bedd0351c37f7813a662b4296a
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/logs/debug-internal.log
@@ -0,0 +1,2239 @@
+2025-01-07 22:06:10,457 INFO    StreamThr :2240208 [internal.py:wandb_internal():85] W&B internal server running at pid: 2240208, started at: 2025-01-07 22:06:10.456569
+2025-01-07 22:06:10,458 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status
+2025-01-07 22:06:10,472 INFO    WriterThread:2240208 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/run-4fmihp69.wandb
+2025-01-07 22:06:10,473 DEBUG   SenderThread:2240208 [sender.py:send():379] send: header
+2025-01-07 22:06:10,481 DEBUG   SenderThread:2240208 [sender.py:send():379] send: run
+2025-01-07 22:06:10,744 INFO    SenderThread:2240208 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files
+2025-01-07 22:06:10,744 INFO    SenderThread:2240208 [sender.py:_start_run_threads():1188] run started: 4fmihp69 with start time 1736305570.457261
+2025-01-07 22:06:10,762 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: check_version
+2025-01-07 22:06:10,763 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: check_version
+2025-01-07 22:06:10,808 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: run_start
+2025-01-07 22:06:10,813 DEBUG   HandlerThread:2240208 [system_info.py:__init__():26] System info init
+2025-01-07 22:06:10,813 DEBUG   HandlerThread:2240208 [system_info.py:__init__():41] System info init done
+2025-01-07 22:06:10,813 INFO    HandlerThread:2240208 [system_monitor.py:start():194] Starting system monitor
+2025-01-07 22:06:10,813 INFO    SystemMonitor:2240208 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-07 22:06:10,813 INFO    SystemMonitor:2240208 [interfaces.py:start():188] Started cpu monitoring
+2025-01-07 22:06:10,814 INFO    HandlerThread:2240208 [system_monitor.py:probe():214] Collecting system info
+2025-01-07 22:06:10,814 INFO    SystemMonitor:2240208 [interfaces.py:start():188] Started disk monitoring
+2025-01-07 22:06:10,814 INFO    SystemMonitor:2240208 [interfaces.py:start():188] Started gpu monitoring
+2025-01-07 22:06:10,815 INFO    SystemMonitor:2240208 [interfaces.py:start():188] Started memory monitoring
+2025-01-07 22:06:10,815 INFO    SystemMonitor:2240208 [interfaces.py:start():188] Started network monitoring
+2025-01-07 22:06:10,826 DEBUG   HandlerThread:2240208 [system_info.py:probe():152] Probing system
+2025-01-07 22:06:10,828 DEBUG   HandlerThread:2240208 [system_info.py:_probe_git():137] Probing git
+2025-01-07 22:06:10,832 DEBUG   HandlerThread:2240208 [system_info.py:_probe_git():145] Probing git done
+2025-01-07 22:06:10,832 DEBUG   HandlerThread:2240208 [system_info.py:probe():200] Probing system done
+2025-01-07 22:06:10,832 DEBUG   HandlerThread:2240208 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T03:06:10.826762', 'startedAt': '2025-01-08T03:06:10.446482', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.0912500000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.994, 'min': 800.0, 'max': 4700.0}, {'current': 4499.94, 'min': 800.0, 'max': 4700.0}, {'current': 4497.561, 'min': 800.0, 'max': 4700.0}, {'current': 4500.022, 'min': 800.0, 'max': 4700.0}, {'current': 4498.542, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 657.744571685791}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-07 22:06:10,832 INFO    HandlerThread:2240208 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-07 22:06:10,832 INFO    HandlerThread:2240208 [system_monitor.py:probe():227] Publishing system info
+2025-01-07 22:06:10,832 DEBUG   HandlerThread:2240208 [system_info.py:_save_code():44] Saving code
+2025-01-07 22:06:10,834 DEBUG   HandlerThread:2240208 [system_info.py:_save_code():67] Saving code done
+2025-01-07 22:06:10,834 DEBUG   HandlerThread:2240208 [system_info.py:_save_patches():84] Saving git patches
+2025-01-07 22:06:10,896 DEBUG   HandlerThread:2240208 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-07 22:06:10,896 INFO    HandlerThread:2240208 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-07 22:06:10,900 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:06:10,900 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-07 22:06:10,900 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-07 22:06:10,900 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-07 22:06:11,143 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/qqyp17un-diff.patch
+2025-01-07 22:06:11,144 INFO    wandb-upload_0:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/0vvacken-wandb-metadata.json
+2025-01-07 22:06:11,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-07 22:06:11,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:06:11,144 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: python_packages
+2025-01-07 22:06:11,145 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:11,145 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:06:11,147 INFO    HandlerThread:2240208 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/tb"
+  save: true
+}
+control {
+  relay_id: "eac7231c22d04ed2b4484c3efc83ccb3"
+}
+_info {
+  stream_id: "4fmihp69"
+}
+
+2025-01-07 22:06:11,147 DEBUG   HandlerThread:2240208 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-07 22:06:11,172 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/ei73ugkb-code/FISH/eval_robot.py
+2025-01-07 22:06:11,267 DEBUG   SenderThread:2240208 [sender.py:send():379] send: telemetry
+2025-01-07 22:06:11,267 DEBUG   SenderThread:2240208 [sender.py:send():379] send: tbrecord
+2025-01-07 22:06:11,268 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:06:11,268 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0 with policy live
+2025-01-07 22:06:11,446 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/wk7l1bwy-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:06:11,746 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/requirements.txt
+2025-01-07 22:06:11,746 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/code/FISH/eval_robot.py
+2025-01-07 22:06:11,746 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/diff.patch
+2025-01-07 22:06:11,746 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:11,746 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:06:11,746 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-metadata.json
+2025-01-07 22:06:11,747 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/code/FISH
+2025-01-07 22:06:11,747 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/code
+2025-01-07 22:06:12,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:13,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:13,745 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:14,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:15,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:16,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:06:16,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:16,468 DEBUG   SenderThread:2240208 [sender.py:send():379] send: config
+2025-01-07 22:06:17,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:17,746 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:18,145 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:19,145 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:19,747 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:20,145 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:21,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:06:21,145 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:21,748 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:22,145 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:23,146 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:23,748 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:24,146 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:25,146 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:25,749 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:26,143 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:06:26,144 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:06:26,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:26,275 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:06:27,147 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:28,147 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:29,147 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:30,147 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:31,147 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:32,145 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:06:32,147 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:33,147 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:33,751 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:34,147 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:35,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:35,752 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:36,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:37,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:37,589 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:06:37,752 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:38,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:39,149 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:40,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:41,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:06:41,144 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:06:41,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:42,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:43,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:43,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:06:43,754 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/config.yaml
+2025-01-07 22:06:44,149 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:45,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:46,149 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:47,149 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:48,150 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:49,150 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:49,249 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:06:49,755 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:50,151 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:51,151 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:52,151 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:53,151 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:53,756 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:54,159 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:54,457 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:06:55,158 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:55,757 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:06:56,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:06:56,144 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:06:56,189 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:57,158 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:58,159 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:06:59,159 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:00,159 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:00,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:01,159 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:02,160 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:03,160 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:04,160 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:04,381 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:07:04,382 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:07:04,382 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_ae65028e83f44b9ac5b1.mp4 with policy now
+2025-01-07 22:07:04,382 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:07:04,382 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:07:04,383 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:07:04,605 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/etwnhs8s-media/videos/eval/0_eval_0_ae65028e83f44b9ac5b1.mp4
+2025-01-07 22:07:04,759 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:07:04,760 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/0_eval_0_ae65028e83f44b9ac5b1.mp4
+2025-01-07 22:07:04,760 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media
+2025-01-07 22:07:04,760 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:07:04,760 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos
+2025-01-07 22:07:05,034 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/nwn9bqku-media/videos/eval/0_eval_0_ae65028e83f44b9ac5b1.mp4
+2025-01-07 22:07:05,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:05,384 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:05,760 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:07:06,165 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:07,161 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:08,161 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:09,161 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:09,761 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:07:10,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:10,469 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:10,815 DEBUG   SystemMonitor:2240208 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-07 22:07:10,816 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:07:11,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:07:11,144 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:07:11,161 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:11,762 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:07:12,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:13,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:14,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:15,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:16,153 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:16,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:17,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:18,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:19,163 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:20,163 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:20,765 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:07:21,017 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/6ib9bsap-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:07:21,154 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:21,163 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:21,765 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:07:22,163 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:23,163 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:23,766 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:07:24,163 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:25,164 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:25,767 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:07:26,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:07:26,144 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:07:26,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:26,256 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:27,164 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:28,164 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:29,164 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:30,169 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:31,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:32,157 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:32,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:33,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:34,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:35,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:36,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:37,158 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:37,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:38,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:39,179 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:40,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:40,817 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:07:41,144 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:07:41,144 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:07:41,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:42,171 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:42,238 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:43,171 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:44,171 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:45,171 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:46,171 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:47,174 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:48,155 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:48,174 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:49,175 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:49,774 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:07:50,174 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:51,175 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:52,177 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:52,277 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:07:52,277 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:07:52,278 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_d5acd6aafe960785647d.mp4 with policy now
+2025-01-07 22:07:52,278 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:07:52,278 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:07:52,278 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:07:52,554 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/pjwmo6o9-media/videos/eval/1_eval_1_d5acd6aafe960785647d.mp4
+2025-01-07 22:07:52,775 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:07:52,776 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/1_eval_1_d5acd6aafe960785647d.mp4
+2025-01-07 22:07:52,776 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:07:53,049 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/4mryielv-media/videos/eval/1_eval_1_d5acd6aafe960785647d.mp4
+2025-01-07 22:07:53,177 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:53,279 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:53,776 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:07:54,180 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:55,180 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:56,147 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:07:56,148 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:07:56,179 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:57,179 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:57,777 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:07:58,181 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:58,413 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:07:59,179 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:07:59,778 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:08:00,180 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:01,179 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:02,180 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:03,181 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:04,157 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:04,181 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:05,182 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:06,181 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:07,181 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:08,181 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:09,182 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:10,156 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:10,182 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:10,817 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:08:11,145 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:08:11,145 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:08:11,189 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:12,182 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:13,182 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:14,182 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:15,182 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:15,259 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:16,183 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:17,184 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:18,184 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:19,186 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:20,184 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:21,157 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:21,184 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:22,184 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:23,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:23,786 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:08:24,033 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/17nqx6vr-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:08:24,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:24,586 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:08:24,587 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:08:24,588 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:08:24,786 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:08:25,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:25,786 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:08:26,145 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:08:26,146 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:08:26,189 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:26,307 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:27,186 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:28,186 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:29,187 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:29,788 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:08:30,187 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:31,187 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:32,157 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:32,187 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:33,187 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:34,187 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:35,187 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:36,188 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:37,158 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:37,188 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:38,188 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:39,188 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:40,188 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:40,818 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:08:41,145 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:08:41,146 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:08:41,189 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:42,190 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:42,385 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:43,189 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:44,190 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:45,190 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:46,190 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:47,190 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:48,158 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:48,190 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:49,190 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:50,191 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:51,191 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:52,191 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:53,159 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:53,191 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:54,191 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:55,193 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:55,729 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:08:55,729 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_537ec4e4ec2357120fef.mp4 with policy now
+2025-01-07 22:08:55,730 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:08:55,731 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:08:55,731 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:08:55,731 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:08:55,798 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:08:55,798 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:08:55,799 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/2_eval_2_537ec4e4ec2357120fef.mp4
+2025-01-07 22:08:55,799 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:08:55,956 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/5w232wyh-media/videos/eval/2_eval_2_537ec4e4ec2357120fef.mp4
+2025-01-07 22:08:56,152 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:08:56,153 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:08:56,192 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:56,219 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/qm16amsm-media/videos/eval/2_eval_2_537ec4e4ec2357120fef.mp4
+2025-01-07 22:08:57,194 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:57,799 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:08:58,197 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:08:58,218 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:08:59,221 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:00,196 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:01,197 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:01,800 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:02,196 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:03,200 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:03,801 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:04,005 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:04,206 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:05,204 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:05,412 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:09:05,413 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:09:05,413 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:09:05,802 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:09:05,802 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:09:06,206 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:07,206 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:07,802 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:08,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:09,192 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:09,233 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:10,206 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:10,818 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:09:11,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:09:11,148 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:09:11,207 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:11,804 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:12,207 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:13,207 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:14,207 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:14,306 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:15,207 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:16,207 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:17,207 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:18,207 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:19,207 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:20,161 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:20,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:21,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:22,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:23,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:24,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:25,161 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:25,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:26,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:09:26,148 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:09:26,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:27,209 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:28,209 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:29,209 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:30,209 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:30,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:31,209 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:32,209 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:33,209 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:34,209 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:35,209 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:36,161 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:36,209 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:37,211 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:37,767 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:09:37,767 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5243c77fb64415fcde38.mp4 with policy now
+2025-01-07 22:09:37,769 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:09:37,770 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:09:37,770 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:09:37,770 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:09:37,813 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:09:37,813 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:37,813 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/3_eval_3_5243c77fb64415fcde38.mp4
+2025-01-07 22:09:37,813 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:09:37,997 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/nul1chui-media/videos/eval/3_eval_3_5243c77fb64415fcde38.mp4
+2025-01-07 22:09:38,213 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:38,256 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/811s36x6-media/videos/eval/3_eval_3_5243c77fb64415fcde38.mp4
+2025-01-07 22:09:39,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:39,813 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:40,216 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:40,819 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:09:41,152 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:09:41,152 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:09:41,223 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:41,305 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:42,222 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:43,221 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:43,815 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:44,214 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:45,216 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:45,815 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:46,216 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:47,166 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:47,218 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:48,224 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:49,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:49,467 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:09:49,467 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:09:49,468 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:09:49,817 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:09:49,817 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:09:50,067 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/k0239dli-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:09:50,222 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:51,224 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:51,817 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:52,167 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:52,224 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:53,221 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:53,818 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:54,219 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:55,219 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:55,819 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:09:56,148 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:09:56,149 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:09:56,219 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:57,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:57,222 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:09:58,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:09:59,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:00,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:01,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:02,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:03,163 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:03,221 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:04,221 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:05,221 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:06,221 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:07,221 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:08,163 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:08,222 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:09,222 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:10,222 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:10,820 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:10:11,149 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:10:11,149 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:10:11,222 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:12,222 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:13,223 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:13,248 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:14,224 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:15,224 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:16,224 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:17,224 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:18,224 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:19,164 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:19,224 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:20,224 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:21,237 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:22,219 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:10:22,219 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_123736b8ecd81d086c47.mp4 with policy now
+2025-01-07 22:10:22,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:10:22,221 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:10:22,221 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:10:22,222 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:10:22,237 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:22,463 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/a01kcc8d-media/videos/eval/4_eval_4_123736b8ecd81d086c47.mp4
+2025-01-07 22:10:22,829 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:10:22,829 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/4_eval_4_123736b8ecd81d086c47.mp4
+2025-01-07 22:10:22,829 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:10:23,132 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/4sm2g1pq-media/videos/eval/4_eval_4_123736b8ecd81d086c47.mp4
+2025-01-07 22:10:23,227 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:23,829 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:10:24,235 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:24,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:25,234 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:26,155 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:10:26,155 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:10:26,243 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:27,244 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:27,831 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:10:28,253 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:29,266 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:29,565 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:29,831 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:10:30,275 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:31,267 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:32,261 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:33,260 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:34,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:35,167 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:35,272 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:36,271 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:37,269 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:37,834 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:10:38,272 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:38,530 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:10:38,530 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:10:38,531 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:10:38,834 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:10:39,269 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:39,835 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:10:40,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:40,820 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:10:40,820 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:41,155 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:10:41,155 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:10:41,275 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:42,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:43,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:43,836 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:10:44,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:45,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:45,841 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:46,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:47,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:48,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:49,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:50,271 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:51,167 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:51,271 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:52,271 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:53,271 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:54,271 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:55,272 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:56,151 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:10:56,152 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:10:56,223 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:10:56,272 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:57,272 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:58,272 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:10:59,272 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:00,272 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:01,272 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:02,167 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:02,273 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:03,273 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:04,273 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:05,273 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:06,273 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:07,274 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:07,625 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:07,845 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:11:08,274 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:09,278 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:09,827 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:11:09,827 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_fdc3dec2e2f95a3c767d.mp4 with policy now
+2025-01-07 22:11:09,828 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:11:09,829 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:11:09,829 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:11:09,830 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:11:09,845 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:11:09,846 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:11:09,846 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/5_eval_5_fdc3dec2e2f95a3c767d.mp4
+2025-01-07 22:11:09,846 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:11:10,047 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/wu3rai74-media/videos/eval/5_eval_5_fdc3dec2e2f95a3c767d.mp4
+2025-01-07 22:11:10,274 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:10,296 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/e4l5q7rx-media/videos/eval/5_eval_5_fdc3dec2e2f95a3c767d.mp4
+2025-01-07 22:11:10,820 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:11:11,161 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:11:11,161 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:11:11,282 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:11,846 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:11:12,296 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:13,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:13,329 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:14,283 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:15,290 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:15,847 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:11:16,279 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:17,284 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:17,848 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:11:18,286 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:19,169 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:19,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:20,300 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:21,304 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:22,296 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:22,584 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:11:22,584 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:11:22,584 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:11:22,850 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:11:22,850 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:11:23,111 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/yk35ljkk-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:11:23,301 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:23,850 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:11:24,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:24,613 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:25,300 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:25,851 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:11:26,161 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:11:26,161 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:11:26,299 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:27,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:27,852 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:11:28,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:29,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:29,694 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:30,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:31,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:32,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:33,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:34,297 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:35,173 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:35,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:36,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:37,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:38,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:39,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:40,175 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:40,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:40,821 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:11:41,155 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:11:41,155 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:11:41,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:42,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:43,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:44,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:45,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:45,312 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:46,298 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:47,299 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:48,299 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:49,299 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:50,299 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:50,869 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:51,299 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:51,861 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:11:52,299 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:53,299 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:54,309 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:54,981 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:11:54,981 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_718698df80de1dc3b1e7.mp4 with policy now
+2025-01-07 22:11:54,991 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:11:54,992 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:11:54,992 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:11:54,992 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:11:55,195 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/wncg1hnt-media/videos/eval/6_eval_6_718698df80de1dc3b1e7.mp4
+2025-01-07 22:11:55,302 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:55,862 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:11:55,863 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:11:55,863 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/6_eval_6_718698df80de1dc3b1e7.mp4
+2025-01-07 22:11:55,863 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:11:56,022 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:11:56,148 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/zdp0t2t3-media/videos/eval/6_eval_6_718698df80de1dc3b1e7.mp4
+2025-01-07 22:11:56,176 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:11:56,176 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:11:56,305 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:57,309 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:58,315 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:59,318 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:11:59,864 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:00,314 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:01,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:01,310 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:02,314 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:03,318 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:03,865 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:04,321 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:05,321 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:06,321 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:06,799 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:06,866 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:12:07,333 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:07,640 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:12:07,640 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:12:07,641 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:12:07,867 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:12:07,867 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:12:07,867 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:08,326 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:09,326 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:09,867 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:10,328 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:10,821 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:12:11,157 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:12:11,157 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:12:11,329 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:11,868 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:12,217 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:12,331 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:13,321 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:13,869 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:14,321 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:15,321 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:16,330 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:17,331 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:18,173 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:18,331 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:19,331 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:20,331 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:21,331 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:22,331 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:23,173 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:23,332 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:24,339 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:25,339 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:25,873 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:26,155 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:12:26,155 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:12:26,339 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:27,339 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:28,273 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:28,340 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:29,340 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:30,340 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:31,340 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:32,340 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:33,340 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:34,173 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:34,341 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:35,342 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:36,342 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:37,342 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:38,342 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:39,174 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:39,342 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:40,360 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:40,822 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:12:41,162 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:12:41,162 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_0c0abd0712ae602c07c3.mp4 with policy now
+2025-01-07 22:12:41,164 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:12:41,164 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:12:41,174 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:12:41,237 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:12:41,237 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:12:41,237 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:12:41,349 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:41,413 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/hzfb5ngp-media/videos/eval/7_eval_7_0c0abd0712ae602c07c3.mp4
+2025-01-07 22:12:41,879 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:12:41,879 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:41,880 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/7_eval_7_0c0abd0712ae602c07c3.mp4
+2025-01-07 22:12:41,880 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:12:42,153 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/qhxnxdkp-media/videos/eval/7_eval_7_0c0abd0712ae602c07c3.mp4
+2025-01-07 22:12:42,362 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:42,880 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:43,375 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:44,238 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:44,365 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:45,371 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:45,881 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:46,369 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:47,374 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:48,372 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:49,384 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:49,498 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:49,882 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:50,372 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:51,371 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:52,380 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:53,382 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:54,375 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:55,196 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:12:55,377 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:56,172 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:12:56,173 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:12:56,384 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:56,700 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:12:56,700 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:12:56,701 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:12:56,885 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:12:56,885 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:12:57,113 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/yjvxbtxq-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:12:57,390 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:57,885 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:12:58,390 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:59,393 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:12:59,886 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:00,417 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:00,532 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:01,398 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:01,887 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:02,400 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:03,392 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:03,888 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:04,392 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:05,392 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:05,541 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:05,889 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:06,393 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:07,393 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:08,393 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:09,393 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:09,890 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:10,393 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:10,822 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:13:10,823 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:11,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:13:11,163 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:13:11,393 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:12,393 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:13,394 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:14,394 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:15,395 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:15,909 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:16,395 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:17,395 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:17,894 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:18,395 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:19,396 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:20,395 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:21,179 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:21,395 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:22,395 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:23,395 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:24,395 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:25,396 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:26,162 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:13:26,163 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:13:26,261 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:26,396 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:27,396 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:28,396 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:29,396 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:30,404 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:30,851 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:13:30,851 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_d8cc872cbcaee100937a.mp4 with policy now
+2025-01-07 22:13:30,855 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:13:30,855 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:13:30,855 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:13:30,855 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:13:30,900 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:13:30,900 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/8_eval_8_d8cc872cbcaee100937a.mp4
+2025-01-07 22:13:30,901 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:13:31,081 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/d26k68mv-media/videos/eval/8_eval_8_d8cc872cbcaee100937a.mp4
+2025-01-07 22:13:31,350 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/9crv7elw-media/videos/eval/8_eval_8_d8cc872cbcaee100937a.mp4
+2025-01-07 22:13:31,428 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:31,900 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:32,179 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:32,426 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:33,404 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:34,408 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:35,403 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:35,902 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:36,414 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:37,403 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:37,903 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:38,093 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:38,427 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:39,426 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:40,409 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:40,823 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:13:41,175 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:13:41,176 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:13:41,430 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:42,421 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:43,319 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:43,411 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:43,905 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:13:44,415 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:44,762 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:13:44,762 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:13:44,762 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:13:44,905 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:13:45,426 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:45,906 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:46,425 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:47,418 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:48,426 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:48,817 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:49,427 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:49,907 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:50,424 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:51,424 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:52,420 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:53,420 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:53,909 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:13:54,030 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:54,420 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:55,421 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:56,165 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:13:56,165 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:13:56,421 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:57,429 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:58,421 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:59,421 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:13:59,430 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:13:59,911 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:00,421 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:01,422 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:01,912 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:02,422 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:03,422 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:04,422 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:05,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:05,422 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:06,422 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:07,422 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:08,422 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:09,422 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:10,423 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:10,823 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:14:10,823 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:11,165 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:14:11,165 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:14:11,423 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:12,423 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:13,423 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:14,423 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:15,423 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:16,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:16,423 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:17,424 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:18,424 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:19,426 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:19,919 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:19,951 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:14:19,951 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_c7576684fb563a572588.mp4 with policy now
+2025-01-07 22:14:19,955 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:14:19,955 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:14:19,955 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:14:19,956 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:14:20,170 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/jz8ps62b-media/videos/eval/9_eval_9_c7576684fb563a572588.mp4
+2025-01-07 22:14:20,444 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:20,919 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:14:20,919 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/9_eval_9_c7576684fb563a572588.mp4
+2025-01-07 22:14:20,920 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:14:21,200 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:21,202 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/rzw69i32-media/videos/eval/9_eval_9_c7576684fb563a572588.mp4
+2025-01-07 22:14:21,439 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:21,919 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:22,457 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:23,443 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:24,450 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:25,459 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:25,921 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:26,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:14:26,185 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:14:26,315 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:26,464 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:27,456 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:27,922 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:28,452 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:29,462 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:30,455 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:31,452 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:32,204 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:32,459 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:32,820 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:14:32,821 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:14:32,821 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:14:32,924 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:14:32,924 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:14:33,165 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/6elnrjsu-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:14:33,489 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:33,924 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:34,470 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:35,463 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:35,925 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:36,479 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:37,474 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:37,913 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:37,926 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:38,483 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:39,480 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:40,475 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:40,824 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:14:41,185 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:14:41,185 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:14:41,479 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:42,464 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:43,085 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:43,464 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:43,928 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:44,464 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:45,464 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:46,464 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:47,465 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:48,194 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:48,465 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:49,465 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:50,465 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:51,467 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:52,467 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:53,466 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:53,933 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:14:54,101 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:54,467 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:55,467 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:56,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:14:56,170 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:14:56,467 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:57,467 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:58,467 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:14:59,305 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:14:59,467 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:00,467 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:01,467 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:02,468 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:03,468 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:04,468 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:05,197 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:05,468 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:06,468 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:07,468 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:08,468 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:09,493 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:09,941 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:10,029 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:15:10,029 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_4e9fbf635a50e60a4cb3.mp4 with policy now
+2025-01-07 22:15:10,069 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:15:10,069 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:15:10,070 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:15:10,070 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:15:10,287 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/ct0xpxq0-media/videos/eval/10_eval_10_4e9fbf635a50e60a4cb3.mp4
+2025-01-07 22:15:10,476 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:10,825 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:15:10,827 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:10,941 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:15:10,942 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/10_eval_10_4e9fbf635a50e60a4cb3.mp4
+2025-01-07 22:15:10,942 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:15:11,213 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:15:11,214 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:15:11,227 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/cft7widw-media/videos/eval/10_eval_10_4e9fbf635a50e60a4cb3.mp4
+2025-01-07 22:15:11,507 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:11,942 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:12,493 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:12,942 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:13,497 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:14,487 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:15,486 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:15,943 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:16,225 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:16,498 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:17,507 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:17,944 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:18,493 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:19,496 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:20,497 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:21,493 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:22,095 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:22,495 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:22,886 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:15:22,886 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:15:22,886 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:15:22,946 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:15:22,946 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:15:23,502 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:23,946 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:24,525 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:25,516 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:26,221 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:15:26,221 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:15:26,500 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:27,306 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:27,520 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:27,948 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:28,508 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:29,518 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:30,510 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:31,508 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:31,949 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:32,508 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:33,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:33,508 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:34,508 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:35,508 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:36,508 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:37,508 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:38,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:38,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:39,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:40,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:40,825 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:15:41,193 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:15:41,193 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:15:41,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:42,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:43,315 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:43,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:44,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:45,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:46,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:47,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:47,956 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:48,377 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:48,510 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:49,510 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:49,956 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:50,510 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:51,510 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:52,510 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:53,510 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:54,220 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:54,510 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:55,510 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:56,193 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:15:56,194 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:15:56,510 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:57,512 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:57,960 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:15:58,515 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:58,605 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:15:58,605 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_6141e16e285aea41414e.mp4 with policy now
+2025-01-07 22:15:58,613 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:15:58,613 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:15:58,613 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:15:58,614 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:15:58,818 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/1f2moux3-media/videos/eval/11_eval_11_6141e16e285aea41414e.mp4
+2025-01-07 22:15:58,960 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:15:58,960 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/11_eval_11_6141e16e285aea41414e.mp4
+2025-01-07 22:15:58,960 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:15:59,226 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/8tz12dd2-media/videos/eval/11_eval_11_6141e16e285aea41414e.mp4
+2025-01-07 22:15:59,518 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:15:59,674 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:15:59,960 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:00,524 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:01,523 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:02,541 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:03,538 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:03,962 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:04,522 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:04,937 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:05,540 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:05,963 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:06,543 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:07,541 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:08,553 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:09,542 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:10,246 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:10,546 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:10,826 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:16:11,199 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:16:11,199 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:16:11,546 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:11,945 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:16:11,945 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:16:11,946 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:16:11,965 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:16:11,965 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:16:12,547 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:12,966 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:16:13,565 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:13,966 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:14,552 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:15,579 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:16,020 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:16,566 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:17,568 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:17,968 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:18,570 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:19,574 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:20,574 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:21,243 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:21,576 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:22,567 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:23,567 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:23,970 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:24,568 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:25,568 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:26,196 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:16:26,196 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:16:26,290 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:26,568 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:27,569 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:28,569 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:29,569 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:30,569 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:31,570 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:31,602 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:32,570 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:33,570 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:33,974 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:34,570 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:35,570 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:36,570 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:37,234 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:37,570 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:37,976 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:38,570 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:39,571 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:40,571 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:40,826 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:16:41,196 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:16:41,196 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:16:41,571 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:42,253 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:42,571 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:43,571 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:44,572 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:45,571 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:46,573 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:47,572 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:47,980 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:48,234 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:48,573 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:49,592 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:49,981 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:50,432 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:16:50,432 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_212b177b80b2034deb4f.mp4 with policy now
+2025-01-07 22:16:50,438 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:16:50,439 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:16:50,439 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:16:50,439 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:16:50,580 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:50,655 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/2iqgnveh-media/videos/eval/12_eval_12_212b177b80b2034deb4f.mp4
+2025-01-07 22:16:50,981 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:16:50,981 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/12_eval_12_212b177b80b2034deb4f.mp4
+2025-01-07 22:16:50,982 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:16:51,269 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/lhqe1hin-media/videos/eval/12_eval_12_212b177b80b2034deb4f.mp4
+2025-01-07 22:16:51,594 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:51,982 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:52,589 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:52,982 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:53,254 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:53,602 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:54,585 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:55,608 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:55,983 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:56,222 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:16:56,223 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:16:56,612 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:57,590 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:57,984 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:16:58,598 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:16:59,249 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:16:59,596 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:00,605 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:01,619 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:01,986 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:17:01,986 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:02,010 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:17:02,010 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:17:02,010 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:17:02,236 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/eh9nqv6k-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:17:02,618 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:02,986 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:17:03,619 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:03,986 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:04,632 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:04,793 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:05,633 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:06,610 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:07,610 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:07,989 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:08,610 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:09,610 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:09,917 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:09,990 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:10,610 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:10,827 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:17:11,201 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:17:11,201 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:17:11,610 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:12,611 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:13,611 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:14,611 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:15,611 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:15,918 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:15,993 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:16,611 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:17,611 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:18,611 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:19,611 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:20,611 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:21,251 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:21,613 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:22,613 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:23,613 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:23,996 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:24,613 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:25,613 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:26,201 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:17:26,201 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:17:26,303 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:26,614 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:27,614 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:27,997 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:28,614 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:29,614 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:29,998 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:30,624 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:31,624 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:31,999 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:32,252 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:32,625 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:33,638 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:34,000 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:34,145 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:17:34,145 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_49e329d2ff2e935251c1.mp4 with policy now
+2025-01-07 22:17:34,156 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:17:34,156 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:17:34,156 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:17:34,156 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:17:34,372 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/s26jid66-media/videos/eval/13_eval_13_49e329d2ff2e935251c1.mp4
+2025-01-07 22:17:34,635 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:35,001 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:17:35,001 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/13_eval_13_49e329d2ff2e935251c1.mp4
+2025-01-07 22:17:35,001 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:17:35,310 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/s9dcdqos-media/videos/eval/13_eval_13_49e329d2ff2e935251c1.mp4
+2025-01-07 22:17:35,654 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:36,001 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:36,666 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:37,002 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:37,256 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:37,647 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:38,647 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:39,649 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:40,003 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:40,653 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:40,828 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:17:41,201 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:17:41,201 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:17:41,664 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:42,004 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:42,348 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:42,660 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:43,664 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:44,665 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:45,663 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:46,676 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:47,663 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:47,926 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:48,006 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:17:48,064 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:17:48,064 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:17:48,064 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:17:48,670 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:49,006 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:17:49,007 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:17:49,659 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:50,007 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:50,672 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:51,663 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:52,690 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:53,373 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:53,673 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:54,009 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:17:54,687 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:55,675 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:56,221 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:17:56,221 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:17:56,703 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:57,697 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:58,704 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:17:59,372 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:17:59,715 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:00,698 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:01,698 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:02,012 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:18:02,698 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:03,698 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:04,698 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:05,262 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:05,698 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:06,698 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:07,698 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:08,014 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:18:08,704 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:09,706 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:10,262 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:10,708 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:10,828 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:18:11,202 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:18:11,203 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:18:11,708 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:12,708 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:13,708 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:14,708 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:15,346 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:15,708 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:16,713 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:17,711 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:18,711 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:19,711 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:20,711 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:21,262 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:21,711 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:22,711 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:23,711 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:24,711 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:25,712 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:26,202 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:18:26,203 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:18:26,281 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:26,712 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:27,723 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:28,022 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:18:28,290 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:18:28,290 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_b1f7698aaf2c3d6e6abf.mp4 with policy now
+2025-01-07 22:18:28,302 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:18:28,303 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:18:28,303 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:18:28,303 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:18:28,513 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/b72q6vqk-media/videos/eval/14_eval_14_b1f7698aaf2c3d6e6abf.mp4
+2025-01-07 22:18:28,715 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:29,023 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:18:29,023 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/14_eval_14_b1f7698aaf2c3d6e6abf.mp4
+2025-01-07 22:18:29,023 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:18:29,310 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/ko4djyjz-media/videos/eval/14_eval_14_b1f7698aaf2c3d6e6abf.mp4
+2025-01-07 22:18:29,741 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:30,023 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:18:30,757 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:31,024 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:18:31,295 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:31,750 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:32,748 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:33,767 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:34,025 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:18:34,770 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:35,750 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:36,026 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:18:36,755 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:37,291 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:37,769 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:38,757 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:39,829 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:40,027 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:18:40,142 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:18:40,142 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:18:40,143 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:18:40,774 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:40,828 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:18:41,028 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:18:41,216 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:18:41,216 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:18:41,766 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:42,028 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:18:42,319 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:42,774 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:43,795 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:44,775 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:45,786 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:46,030 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:18:46,812 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:47,791 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:48,286 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:48,783 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:49,813 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:50,837 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:51,811 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:52,807 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:53,416 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:53,806 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:54,033 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:18:54,806 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:55,807 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:56,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:18:56,208 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:18:56,807 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:57,807 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:58,807 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:18:59,360 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:18:59,807 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:00,808 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:01,808 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:02,808 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:03,808 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:04,809 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:05,277 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:05,809 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:06,809 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:07,809 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:08,809 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:09,809 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:10,278 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:10,809 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:10,829 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:19:11,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:19:11,209 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:19:11,809 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:12,809 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:13,810 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:14,810 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:15,361 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:15,810 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:16,810 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:17,810 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:18,810 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:19,839 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:20,044 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:19:20,474 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:20,637 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:19:20,637 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_b314c0181db144f9c4bb.mp4 with policy now
+2025-01-07 22:19:20,656 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:19:20,657 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:19:20,657 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:19:20,658 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:19:20,822 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:20,880 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/gweu51ng-media/videos/eval/15_eval_15_b314c0181db144f9c4bb.mp4
+2025-01-07 22:19:21,044 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:19:21,044 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/15_eval_15_b314c0181db144f9c4bb.mp4
+2025-01-07 22:19:21,045 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:19:21,363 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/gm3ucvrw-media/videos/eval/15_eval_15_b314c0181db144f9c4bb.mp4
+2025-01-07 22:19:21,841 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:22,045 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:19:22,852 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:23,045 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:19:23,851 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:24,858 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:25,874 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:26,046 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:19:26,223 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:19:26,223 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:19:26,374 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:26,889 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:27,849 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:28,047 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:19:28,865 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:29,869 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:30,867 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:31,879 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:32,290 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:32,876 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:33,883 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:34,884 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:35,894 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:36,892 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:37,051 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:19:37,211 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:19:37,211 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:19:37,212 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:19:37,295 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/nvgsxp4i-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:19:37,489 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:37,896 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:38,052 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:19:38,052 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:19:38,908 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:39,905 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:40,829 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:19:40,891 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:41,230 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:19:41,230 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:19:41,904 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:42,053 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:19:42,901 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:43,356 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:43,916 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:44,902 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:45,902 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:46,898 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:47,899 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:48,899 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:49,056 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:19:49,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:49,899 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:50,899 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:51,899 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:52,899 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:53,899 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:54,296 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:54,899 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:55,899 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:56,216 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:19:56,217 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:19:56,900 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:57,900 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:58,900 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:19:59,304 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:19:59,909 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:00,909 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:01,909 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:02,910 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:03,909 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:04,910 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:05,296 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:20:05,910 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:06,911 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:07,911 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:08,911 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:09,911 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:10,301 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:20:10,830 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:20:10,911 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:11,217 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:20:11,217 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:20:11,912 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:12,912 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:13,912 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:14,936 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:15,313 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:20:15,417 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:20:15,417 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_a93b394f6e07a47f91ce.mp4 with policy now
+2025-01-07 22:20:15,427 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:20:15,428 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:20:15,428 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:20:15,428 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:20:15,644 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/0qjdem3p-media/videos/eval/16_eval_16_a93b394f6e07a47f91ce.mp4
+2025-01-07 22:20:15,928 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:16,073 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:20:16,073 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/16_eval_16_a93b394f6e07a47f91ce.mp4
+2025-01-07 22:20:16,073 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:20:16,360 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/3copf12h-media/videos/eval/16_eval_16_a93b394f6e07a47f91ce.mp4
+2025-01-07 22:20:16,933 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:17,074 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:20:17,939 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:18,953 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:19,968 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:20,945 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:20,953 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:20:21,075 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:20:21,945 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:22,941 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:23,951 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:24,968 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:25,077 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:20:25,952 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:26,232 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:20:26,232 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:20:26,344 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:20:26,973 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:27,965 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:28,972 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:29,995 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:30,977 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:31,990 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:32,051 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:20:32,080 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:20:32,276 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:20:32,276 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:20:32,276 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:20:32,983 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:33,081 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:20:33,081 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:20:33,984 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:34,971 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:35,082 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:20:35,987 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:36,973 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:37,537 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:20:37,999 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:39,008 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:39,083 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:20:39,996 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:40,830 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:20:41,010 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:41,223 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:20:41,223 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:20:42,024 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:43,033 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:43,328 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:20:44,006 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:45,002 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:46,003 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:47,003 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:47,087 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:20:48,003 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:49,003 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:49,308 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:20:50,003 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:51,003 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:52,004 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:53,004 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:54,004 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:54,309 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:20:55,004 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:56,005 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:56,217 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:20:56,217 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:20:57,005 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:58,005 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:20:59,006 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:00,005 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:00,308 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:01,005 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:02,005 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:03,005 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:04,006 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:05,006 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:05,308 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:06,006 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:07,006 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:08,006 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:09,006 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:10,007 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:10,308 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:10,831 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:21:11,007 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:11,217 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:21:11,217 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:21:12,021 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:13,019 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:13,098 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:21:13,235 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:21:13,235 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_e673279ceb422c546539.mp4 with policy now
+2025-01-07 22:21:13,241 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:21:13,248 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:21:13,248 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:21:13,248 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:21:13,461 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/ohhgn9jr-media/videos/eval/17_eval_17_e673279ceb422c546539.mp4
+2025-01-07 22:21:14,025 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:14,098 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:21:14,098 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/17_eval_17_e673279ceb422c546539.mp4
+2025-01-07 22:21:14,099 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:21:14,386 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/stkhwb1n-media/videos/eval/17_eval_17_e673279ceb422c546539.mp4
+2025-01-07 22:21:15,035 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:15,099 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:21:15,316 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:16,046 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:17,054 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:18,042 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:19,039 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:19,101 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:21:20,031 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:20,969 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:21,036 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:22,061 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:23,060 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:23,102 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:21:24,064 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:25,051 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:26,060 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:26,241 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:21:26,241 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:21:26,360 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:27,091 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:27,340 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:21:27,340 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:21:27,341 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:21:28,105 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:21:28,105 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:21:28,115 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:29,105 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:29,105 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:21:30,081 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:31,096 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:31,106 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:21:31,657 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:32,082 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:33,081 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:33,107 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:21:34,081 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:35,081 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:35,108 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:21:36,082 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:37,082 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:37,320 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:38,082 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:39,082 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:40,082 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:40,831 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:21:41,083 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:41,219 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:21:41,220 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:21:42,083 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:43,083 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:43,320 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:44,083 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:45,083 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:46,084 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:47,084 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:48,084 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:48,320 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:49,084 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:50,084 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:51,085 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:52,085 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:53,085 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:53,325 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:54,085 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:55,085 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:56,085 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:56,219 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:21:56,220 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:21:57,085 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:57,119 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:21:58,085 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:58,370 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:21:59,086 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:21:59,120 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:00,112 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:00,414 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:22:00,414 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_fd5b9c7ce55fe4462130.mp4 with policy now
+2025-01-07 22:22:00,417 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:22:00,418 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:22:00,418 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:22:00,418 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:22:00,639 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/o1vdcb9z-media/videos/eval/18_eval_18_fd5b9c7ce55fe4462130.mp4
+2025-01-07 22:22:01,102 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:01,121 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:22:01,121 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:01,121 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/18_eval_18_fd5b9c7ce55fe4462130.mp4
+2025-01-07 22:22:01,121 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:22:01,411 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/ppuctjtu-media/videos/eval/18_eval_18_fd5b9c7ce55fe4462130.mp4
+2025-01-07 22:22:02,095 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:03,095 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:03,122 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:04,096 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:04,124 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:22:05,116 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:06,121 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:07,124 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:07,130 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:08,125 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:09,115 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:09,124 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:09,389 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:22:10,112 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:10,831 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:22:11,131 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:11,226 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:22:11,226 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:22:12,124 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:12,126 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:22:12,397 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:22:12,398 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:22:12,398 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:22:13,126 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:22:13,126 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:13,143 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:14,122 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:14,721 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:22:15,127 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:15,150 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:16,118 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:17,121 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:17,128 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:18,121 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:19,125 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:19,129 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:20,125 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:20,352 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:22:21,126 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:22,126 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:23,126 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:24,126 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:25,127 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:26,127 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:26,222 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:22:26,222 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:22:26,366 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:22:27,127 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:28,127 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:29,133 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:30,133 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:31,133 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:32,133 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:32,341 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:22:33,134 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:34,133 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:35,134 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:36,134 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:37,135 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:38,135 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:38,341 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:22:39,135 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:40,135 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:40,832 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:22:41,135 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:41,138 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:41,225 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:22:41,225 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:22:42,135 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:43,139 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:43,139 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:43,764 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:22:44,152 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:45,140 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:45,149 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:45,374 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:22:45,374 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_b857c64ab339c17adb15.mp4 with policy now
+2025-01-07 22:22:45,378 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:22:45,378 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:22:45,378 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:22:45,378 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:22:45,602 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/z1tye7wm-media/videos/eval/19_eval_19_b857c64ab339c17adb15.mp4
+2025-01-07 22:22:46,141 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:22:46,141 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/19_eval_19_b857c64ab339c17adb15.mp4
+2025-01-07 22:22:46,141 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval
+2025-01-07 22:22:46,192 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:46,430 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/89phxfif-media/videos/eval/19_eval_19_b857c64ab339c17adb15.mp4
+2025-01-07 22:22:47,141 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:47,170 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:48,165 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:48,893 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:22:49,161 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:50,171 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:51,143 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:51,178 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:52,177 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:53,177 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:54,184 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:54,337 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:22:55,145 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:22:55,236 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:56,182 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:56,264 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:22:56,265 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:22:57,195 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:58,208 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:59,197 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:22:59,395 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:00,211 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:01,213 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:02,237 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:03,241 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:04,253 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:05,248 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:05,389 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:06,269 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:07,252 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:08,270 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:09,290 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:10,267 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:10,401 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:10,832 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:23:11,260 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:23:11,261 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:23:11,307 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:12,289 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:13,321 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:14,304 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:15,311 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:16,295 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:16,388 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:17,300 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:18,316 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:19,335 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:20,310 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:21,307 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:21,403 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:22,317 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:23,304 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:24,324 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:25,317 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:26,251 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:23:26,251 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:23:26,327 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:26,483 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:27,319 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:28,316 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:29,347 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:30,357 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:31,361 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:32,356 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:32,461 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:33,361 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:34,351 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:35,339 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:36,366 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:37,379 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:37,611 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:38,163 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:23:38,364 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:38,418 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/12cip9vv-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:23:38,502 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:23:38,502 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:38,502 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:39,164 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:23:39,164 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:23:39,388 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:39,520 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 22:23:39,521 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 22:23:39,883 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:23:39,931 INFO    wandb-upload_2:2240208 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyshrfqyx
+2025-01-07 22:23:40,164 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json
+2025-01-07 22:23:40,165 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media
+2025-01-07 22:23:40,165 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval
+2025-01-07 22:23:40,165 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table
+2025-01-07 22:23:40,328 INFO    SenderThread:2240208 [sender.py:send_request_log_artifact():1525] logged artifact run-4fmihp69-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzI0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg5MDE1OQ==', 'latestArtifact': None}}
+2025-01-07 22:23:40,328 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:23:40,328 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json with policy now
+2025-01-07 22:23:40,328 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:23:40,328 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,329 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,329 DEBUG   SenderThread:2240208 [sender.py:send():379] send: telemetry
+2025-01-07 22:23:40,329 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,329 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,329 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,330 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,330 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,330 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,331 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,331 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,331 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,331 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,332 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,332 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,332 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,332 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,333 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,333 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,333 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,334 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,334 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,334 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,334 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:40,335 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:40,347 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 22:23:40,347 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 22:23:40,358 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 22:23:40,359 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:40,500 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/3p0039k3-media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json
+2025-01-07 22:23:40,738 INFO    wandb-upload_1:2240208 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphvl7ccyh
+2025-01-07 22:23:40,761 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/ho7k69m8-media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json
+2025-01-07 22:23:41,165 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:23:41,165 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:23:41,165 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval/total_success_rate_ci_21_54cfd2c4c0340bc54565.table.json
+2025-01-07 22:23:41,165 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval
+2025-01-07 22:23:41,202 INFO    SenderThread:2240208 [sender.py:send_request_log_artifact():1525] logged artifact run-4fmihp69-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzI2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg5MDE2OA==', 'latestArtifact': None}}
+2025-01-07 22:23:41,202 DEBUG   SenderThread:2240208 [sender.py:send():379] send: files
+2025-01-07 22:23:41,202 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_54cfd2c4c0340bc54565.table.json with policy now
+2025-01-07 22:23:41,202 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:23:41,203 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:41,204 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:41,204 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:23:41,240 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 22:23:41,240 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 22:23:41,359 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:41,385 INFO    wandb-upload_0:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/p9atv43p-media/table/eval/total_success_rate_ci_21_54cfd2c4c0340bc54565.table.json
+2025-01-07 22:23:41,643 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/3sihhq5w-media/table/eval/total_success_rate_ci_21_54cfd2c4c0340bc54565.table.json
+2025-01-07 22:23:42,165 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:23:42,359 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:43,166 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:23:43,360 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:43,450 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:44,360 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:45,360 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:46,360 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:47,361 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:47,750 DEBUG   SenderThread:2240208 [sender.py:send():379] send: exit
+2025-01-07 22:23:47,750 INFO    SenderThread:2240208 [sender.py:send_exit():586] handling exit code: 0
+2025-01-07 22:23:47,751 INFO    SenderThread:2240208 [sender.py:send_exit():588] handling runtime: 1056
+2025-01-07 22:23:47,751 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:47,751 INFO    SenderThread:2240208 [sender.py:send_exit():594] send defer
+2025-01-07 22:23:47,751 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:47,751 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-07 22:23:47,751 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:47,752 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-07 22:23:47,752 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 1
+2025-01-07 22:23:47,752 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:47,752 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-07 22:23:47,752 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:47,752 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-07 22:23:47,752 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 2
+2025-01-07 22:23:47,752 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:47,752 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-07 22:23:47,752 INFO    HandlerThread:2240208 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-07 22:23:47,752 DEBUG   SystemMonitor:2240208 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-07 22:23:47,753 INFO    HandlerThread:2240208 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-07 22:23:47,753 DEBUG   SystemMonitor:2240208 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-07 22:23:47,753 INFO    HandlerThread:2240208 [interfaces.py:finish():200] Joined disk monitor
+2025-01-07 22:23:47,806 INFO    HandlerThread:2240208 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-07 22:23:47,806 INFO    HandlerThread:2240208 [interfaces.py:finish():200] Joined memory monitor
+2025-01-07 22:23:47,806 INFO    HandlerThread:2240208 [interfaces.py:finish():200] Joined network monitor
+2025-01-07 22:23:47,807 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:47,807 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-07 22:23:47,807 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 3
+2025-01-07 22:23:47,807 DEBUG   SenderThread:2240208 [sender.py:send():379] send: stats
+2025-01-07 22:23:47,807 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:47,807 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-07 22:23:47,807 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:47,807 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-07 22:23:47,808 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 4
+2025-01-07 22:23:47,808 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:47,808 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-07 22:23:48,169 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:23:49,169 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:23:54,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 22:23:54,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:54,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-07 22:23:54,509 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:54,510 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:54,510 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-07 22:23:54,510 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 5
+2025-01-07 22:23:54,510 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 22:23:54,511 DEBUG   SenderThread:2240208 [sender.py:send():379] send: history
+2025-01-07 22:23:54,511 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 22:23:54,511 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:54,511 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-07 22:23:54,512 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:54,512 DEBUG   SenderThread:2240208 [sender.py:send():379] send: summary
+2025-01-07 22:23:54,516 INFO    SenderThread:2240208 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 22:23:54,516 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:54,516 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-07 22:23:54,516 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 6
+2025-01-07 22:23:54,516 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:54,516 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-07 22:23:54,516 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:54,516 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-07 22:23:54,520 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 22:23:54,637 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 7
+2025-01-07 22:23:54,638 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:54,638 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-07 22:23:54,638 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:54,638 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-07 22:23:54,747 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 22:23:55,172 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/config.yaml
+2025-01-07 22:23:55,172 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:23:56,568 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 8
+2025-01-07 22:23:56,568 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 22:23:56,568 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:56,568 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-07 22:23:56,569 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:56,569 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-07 22:23:56,569 INFO    SenderThread:2240208 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-07 22:23:56,572 INFO    SenderThread:2240208 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-07 22:23:56,574 INFO    SenderThread:2240208 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-07 22:23:56,589 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 9
+2025-01-07 22:23:56,589 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:56,589 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-07 22:23:56,589 DEBUG   SenderThread:2240208 [sender.py:send():379] send: artifact
+2025-01-07 22:23:56,747 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 22:23:56,972 INFO    wandb-upload_1:2240208 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk6rt9n13
+2025-01-07 22:23:57,068 INFO    wandb-upload_0:2240208 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpeiw4vr0r
+2025-01-07 22:23:57,107 INFO    wandb-upload_2:2240208 [upload_job.py:push():88] Uploaded file /tmp/tmpphgzzj5s/wandb-job.json
+2025-01-07 22:23:57,173 INFO    Thread-12 :2240208 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:23:57,646 INFO    SenderThread:2240208 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjUyMjY3OA==', 'versionIndex': 438}}}
+2025-01-07 22:23:57,646 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:57,646 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-07 22:23:57,646 INFO    SenderThread:2240208 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-07 22:23:58,174 INFO    SenderThread:2240208 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files
+2025-01-07 22:23:58,174 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/config.yaml config.yaml
+2025-01-07 22:23:58,175 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json wandb-summary.json
+2025-01-07 22:23:58,175 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/requirements.txt requirements.txt
+2025-01-07 22:23:58,175 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-metadata.json wandb-metadata.json
+2025-01-07 22:23:58,175 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0 events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:23:58,175 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log output.log
+2025-01-07 22:23:58,180 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/diff.patch diff.patch
+2025-01-07 22:23:58,182 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/11_eval_11_6141e16e285aea41414e.mp4 media/videos/eval/11_eval_11_6141e16e285aea41414e.mp4
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/15_eval_15_b314c0181db144f9c4bb.mp4 media/videos/eval/15_eval_15_b314c0181db144f9c4bb.mp4
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/0_eval_0_ae65028e83f44b9ac5b1.mp4 media/videos/eval/0_eval_0_ae65028e83f44b9ac5b1.mp4
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/9_eval_9_c7576684fb563a572588.mp4 media/videos/eval/9_eval_9_c7576684fb563a572588.mp4
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/2_eval_2_537ec4e4ec2357120fef.mp4 media/videos/eval/2_eval_2_537ec4e4ec2357120fef.mp4
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/19_eval_19_b857c64ab339c17adb15.mp4 media/videos/eval/19_eval_19_b857c64ab339c17adb15.mp4
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/3_eval_3_5243c77fb64415fcde38.mp4 media/videos/eval/3_eval_3_5243c77fb64415fcde38.mp4
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/5_eval_5_fdc3dec2e2f95a3c767d.mp4 media/videos/eval/5_eval_5_fdc3dec2e2f95a3c767d.mp4
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/7_eval_7_0c0abd0712ae602c07c3.mp4 media/videos/eval/7_eval_7_0c0abd0712ae602c07c3.mp4
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/12_eval_12_212b177b80b2034deb4f.mp4 media/videos/eval/12_eval_12_212b177b80b2034deb4f.mp4
+2025-01-07 22:23:58,183 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/17_eval_17_e673279ceb422c546539.mp4 media/videos/eval/17_eval_17_e673279ceb422c546539.mp4
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/16_eval_16_a93b394f6e07a47f91ce.mp4 media/videos/eval/16_eval_16_a93b394f6e07a47f91ce.mp4
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/10_eval_10_4e9fbf635a50e60a4cb3.mp4 media/videos/eval/10_eval_10_4e9fbf635a50e60a4cb3.mp4
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/14_eval_14_b1f7698aaf2c3d6e6abf.mp4 media/videos/eval/14_eval_14_b1f7698aaf2c3d6e6abf.mp4
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/6_eval_6_718698df80de1dc3b1e7.mp4 media/videos/eval/6_eval_6_718698df80de1dc3b1e7.mp4
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/1_eval_1_d5acd6aafe960785647d.mp4 media/videos/eval/1_eval_1_d5acd6aafe960785647d.mp4
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/18_eval_18_fd5b9c7ce55fe4462130.mp4 media/videos/eval/18_eval_18_fd5b9c7ce55fe4462130.mp4
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/13_eval_13_49e329d2ff2e935251c1.mp4 media/videos/eval/13_eval_13_49e329d2ff2e935251c1.mp4
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/8_eval_8_d8cc872cbcaee100937a.mp4 media/videos/eval/8_eval_8_d8cc872cbcaee100937a.mp4
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/videos/eval/4_eval_4_123736b8ecd81d086c47.mp4 media/videos/eval/4_eval_4_123736b8ecd81d086c47.mp4
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval/total_success_rate_ci_21_54cfd2c4c0340bc54565.table.json media/table/eval/total_success_rate_ci_21_54cfd2c4c0340bc54565.table.json
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json
+2025-01-07 22:23:58,184 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 10
+2025-01-07 22:23:58,184 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 22:23:58,184 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:58,185 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-07 22:23:58,185 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:58,185 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-07 22:23:58,185 INFO    SenderThread:2240208 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 22:23:58,353 INFO    wandb-upload_0:2240208 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/wandb-summary.json
+2025-01-07 22:23:58,365 INFO    wandb-upload_1:2240208 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/config.yaml
+2025-01-07 22:23:58,409 INFO    wandb-upload_2:2240208 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/requirements.txt
+2025-01-07 22:23:58,450 INFO    wandb-upload_4:2240208 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/files/output.log
+2025-01-07 22:23:58,526 INFO    wandb-upload_3:2240208 [upload_job.py:push():130] Uploaded file /tmp/tmp49f3m96uwandb/biyb826f-events.out.tfevents.1736305571.leonmkim-ROG-Strix-G15CS-G15CS.2240043.0
+2025-01-07 22:23:58,727 INFO    Thread-11 (_thread_body):2240208 [sender.py:transition_state():614] send defer: 11
+2025-01-07 22:23:58,727 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:58,727 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-07 22:23:58,727 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:58,728 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-07 22:23:58,728 INFO    SenderThread:2240208 [file_pusher.py:join():175] waiting for file pusher
+2025-01-07 22:23:58,731 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 12
+2025-01-07 22:23:58,731 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:58,731 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-07 22:23:58,731 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:58,731 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-07 22:23:58,731 INFO    SenderThread:2240208 [file_stream.py:finish():601] file stream finish called
+2025-01-07 22:23:58,748 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 22:23:58,935 INFO    SenderThread:2240208 [file_stream.py:finish():605] file stream finish is done
+2025-01-07 22:23:58,935 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 13
+2025-01-07 22:23:58,936 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 22:23:58,936 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:58,936 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-07 22:23:58,936 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:58,936 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-07 22:23:58,936 INFO    SenderThread:2240208 [sender.py:transition_state():614] send defer: 14
+2025-01-07 22:23:58,936 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 22:23:58,936 DEBUG   SenderThread:2240208 [sender.py:send():379] send: final
+2025-01-07 22:23:58,937 INFO    HandlerThread:2240208 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-07 22:23:58,937 DEBUG   SenderThread:2240208 [sender.py:send():379] send: footer
+2025-01-07 22:23:58,937 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: defer
+2025-01-07 22:23:58,937 INFO    SenderThread:2240208 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-07 22:23:58,937 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 22:23:58,937 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: server_info
+2025-01-07 22:23:58,937 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 22:23:58,938 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: server_info
+2025-01-07 22:23:58,939 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 22:23:58,939 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-07 22:23:58,940 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-07 22:23:58,941 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 22:23:59,004 DEBUG   SenderThread:2240208 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 22:23:59,004 INFO    MainThread:2240208 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-07 22:23:59,005 INFO    MainThread:2240208 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-07 22:23:59,005 INFO    MainThread:2240208 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-07 22:23:59,005 DEBUG   HandlerThread:2240208 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-07 22:23:59,005 INFO    HandlerThread:2240208 [handler.py:finish():882] shutting down handler
+2025-01-07 22:23:59,939 INFO    WriterThread:2240208 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/run-4fmihp69.wandb
+2025-01-07 22:24:00,004 INFO    SenderThread:2240208 [sender.py:finish():1615] shutting down sender
+2025-01-07 22:24:00,004 INFO    SenderThread:2240208 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 22:24:00,004 INFO    SenderThread:2240208 [file_pusher.py:join():175] waiting for file pusher
diff --git a/4fmihp69/wandb/run-20250107_220610-4fmihp69/logs/debug.log b/4fmihp69/wandb/run-20250107_220610-4fmihp69/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..1208a09c17fa0d0ef2dcc483d6bb9039b46833e0
--- /dev/null
+++ b/4fmihp69/wandb/run-20250107_220610-4fmihp69/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_setup.py:_flush():76] Configure stats pid to 2240043
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/settings
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/logs/debug.log
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/wandb/run-20250107_220610-4fmihp69/logs/debug-internal.log
+2025-01-07 22:06:10,448 INFO    MainThread:2240043 [wandb_init.py:init():569] calling init triggers
+2025-01-07 22:06:10,449 INFO    MainThread:2240043 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69'}
+2025-01-07 22:06:10,449 INFO    MainThread:2240043 [wandb_init.py:init():619] starting backend
+2025-01-07 22:06:10,449 INFO    MainThread:2240043 [wandb_init.py:init():623] setting up manager
+2025-01-07 22:06:10,456 INFO    MainThread:2240043 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-07 22:06:10,457 INFO    MainThread:2240043 [wandb_init.py:init():631] backend started and connected
+2025-01-07 22:06:10,472 INFO    MainThread:2240043 [wandb_init.py:init():720] updated telemetry
+2025-01-07 22:06:10,479 INFO    MainThread:2240043 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-07 22:06:10,762 INFO    MainThread:2240043 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-07 22:06:10,803 INFO    MainThread:2240043 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-07 22:06:10,803 INFO    MainThread:2240043 [wandb_init.py:init():804] starting run threads in backend
+2025-01-07 22:06:11,143 INFO    MainThread:2240043 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-07 22:06:11,144 INFO    MainThread:2240043 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-07 22:06:11,144 INFO    MainThread:2240043 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-07 22:06:11,144 INFO    MainThread:2240043 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-07 22:06:11,146 INFO    MainThread:2240043 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-07 22:06:11,146 INFO    MainThread:2240043 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4fmihp69/tb, True
+2025-01-07 22:06:16,468 INFO    MainThread:2240043 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-07 22:24:00,006 WARNING MsgRouterThr:2240043 [router.py:message_loop():77] message_loop has been closed
diff --git a/4r9mcmxe/.hydra/config.yaml b/4r9mcmxe/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804
--- /dev/null
+++ b/4r9mcmxe/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1002_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/4r9mcmxe/.hydra/hydra.yaml b/4r9mcmxe/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3ff6df556231e32b1af79541dcc7253cad737599
--- /dev/null
+++ b/4r9mcmxe/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/4r9mcmxe
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/4r9mcmxe/.hydra/overrides.yaml b/4r9mcmxe/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/4r9mcmxe/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/4r9mcmxe/eval_policy.log b/4r9mcmxe/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..837de0b99f17c0588bd119d70599f38b6e42e412
--- /dev/null
+++ b/4r9mcmxe/eval_policy.log
@@ -0,0 +1,15 @@
+[2025-01-07 21:43:24,810][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-07 21:43:24,813][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-07 21:43:28,293][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+
diff --git a/bkletf5j/.hydra/config.yaml b/bkletf5j/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804
--- /dev/null
+++ b/bkletf5j/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1002_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/bkletf5j/.hydra/overrides.yaml b/bkletf5j/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/bkletf5j/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/bkletf5j/episode_rosbags/aligned_depth_to_color_K.npy b/bkletf5j/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/bkletf5j/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/bkletf5j/episode_rosbags/cam_tf_world.npy b/bkletf5j/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/bkletf5j/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/bkletf5j/episode_rosbags/color_K.npy b/bkletf5j/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/bkletf5j/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/bkletf5j/episode_rosbags/depth_K.npy b/bkletf5j/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/bkletf5j/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/bkletf5j/eval_robot.log b/bkletf5j/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..d47a7e6cdcca02e2a1ba200484d4cbe62e0e5cab
--- /dev/null
+++ b/bkletf5j/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-06 20:49:58,000][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-06 20:49:58,003][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/bkletf5j/eval_video/0_eval.mp4 b/bkletf5j/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..0b9fe80e3257d592d3ff3d0d7f162abad2cdaa9b
--- /dev/null
+++ b/bkletf5j/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5629c4a8c29fc9b2a6bcd5b52045b248fa99a988b49da528172799d54ae9e43
+size 1138009
diff --git a/bkletf5j/eval_video/1_eval.mp4 b/bkletf5j/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..dfe1619b871855b91927fe6b690d51ff298b0bea
--- /dev/null
+++ b/bkletf5j/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0cb5720b5b83600f05130a6d5d7c655d93900882d84fc0a0fcb4fc77ed3ee7a
+size 1135045
diff --git a/bkletf5j/eval_video/2_eval.mp4 b/bkletf5j/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..581bcb00885fab23c514e284e49d008921d20b6d
--- /dev/null
+++ b/bkletf5j/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cf0e0208d2fc5ae593796264b65c54bc47568501b27870d0019ac790e381dd1
+size 1137418
diff --git a/bkletf5j/eval_video/3_eval.mp4 b/bkletf5j/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1cc1459b21506352e313e09f574fccd7e87883fa
--- /dev/null
+++ b/bkletf5j/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f993ed9f4bf0eae5cf19f3476f8fdd87803f51f4afc6fb30ec1d84255a3e003
+size 1130405
diff --git a/bkletf5j/eval_video/4_eval.mp4 b/bkletf5j/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..44e45bfc1b7a85fe35c98e20a7ceb6efe9f351db
--- /dev/null
+++ b/bkletf5j/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a8935b2358637140fbc934be6af895927356f6549f7c53e6b407ab74efdc608
+size 1186142
diff --git a/bkletf5j/tb/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0 b/bkletf5j/tb/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
new file mode 100644
index 0000000000000000000000000000000000000000..03570e4165c49a32748d63aaae36c3b7734c3cc8
--- /dev/null
+++ b/bkletf5j/tb/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f649be3e9b01130a1db0ebdbe6002bc8d477288decda4329be4e8154d86113bf
+size 6539
diff --git a/bkletf5j/wandb/debug-internal.log b/bkletf5j/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..58f6bf74ea362f4b0ee338d3997c049de51b3430
--- /dev/null
+++ b/bkletf5j/wandb/debug-internal.log
@@ -0,0 +1,748 @@
+2025-01-06 20:50:04,134 INFO    StreamThr :2115858 [internal.py:wandb_internal():85] W&B internal server running at pid: 2115858, started at: 2025-01-06 20:50:04.134315
+2025-01-06 20:50:04,135 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status
+2025-01-06 20:50:04,146 INFO    WriterThread:2115858 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/run-bkletf5j.wandb
+2025-01-06 20:50:04,147 DEBUG   SenderThread:2115858 [sender.py:send():379] send: header
+2025-01-06 20:50:04,155 DEBUG   SenderThread:2115858 [sender.py:send():379] send: run
+2025-01-06 20:50:04,386 INFO    SenderThread:2115858 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files
+2025-01-06 20:50:04,386 INFO    SenderThread:2115858 [sender.py:_start_run_threads():1188] run started: bkletf5j with start time 1736214604.134792
+2025-01-06 20:50:04,401 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: check_version
+2025-01-06 20:50:04,401 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: check_version
+2025-01-06 20:50:04,475 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: run_start
+2025-01-06 20:50:04,483 DEBUG   HandlerThread:2115858 [system_info.py:__init__():26] System info init
+2025-01-06 20:50:04,483 DEBUG   HandlerThread:2115858 [system_info.py:__init__():41] System info init done
+2025-01-06 20:50:04,483 INFO    HandlerThread:2115858 [system_monitor.py:start():194] Starting system monitor
+2025-01-06 20:50:04,483 INFO    SystemMonitor:2115858 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-06 20:50:04,483 INFO    HandlerThread:2115858 [system_monitor.py:probe():214] Collecting system info
+2025-01-06 20:50:04,485 INFO    SystemMonitor:2115858 [interfaces.py:start():188] Started cpu monitoring
+2025-01-06 20:50:04,485 INFO    SystemMonitor:2115858 [interfaces.py:start():188] Started disk monitoring
+2025-01-06 20:50:04,486 INFO    SystemMonitor:2115858 [interfaces.py:start():188] Started gpu monitoring
+2025-01-06 20:50:04,487 INFO    SystemMonitor:2115858 [interfaces.py:start():188] Started memory monitoring
+2025-01-06 20:50:04,487 INFO    SystemMonitor:2115858 [interfaces.py:start():188] Started network monitoring
+2025-01-06 20:50:04,498 DEBUG   HandlerThread:2115858 [system_info.py:probe():152] Probing system
+2025-01-06 20:50:04,499 DEBUG   HandlerThread:2115858 [system_info.py:_probe_git():137] Probing git
+2025-01-06 20:50:04,504 DEBUG   HandlerThread:2115858 [system_info.py:_probe_git():145] Probing git done
+2025-01-06 20:50:04,504 DEBUG   HandlerThread:2115858 [system_info.py:probe():200] Probing system done
+2025-01-06 20:50:04,504 DEBUG   HandlerThread:2115858 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:50:04.498228', 'startedAt': '2025-01-07T01:50:04.126209', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.57975, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4495.894, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 525.3030891418457}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-06 20:50:04,504 INFO    HandlerThread:2115858 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-06 20:50:04,504 INFO    HandlerThread:2115858 [system_monitor.py:probe():227] Publishing system info
+2025-01-06 20:50:04,504 DEBUG   HandlerThread:2115858 [system_info.py:_save_code():44] Saving code
+2025-01-06 20:50:04,506 DEBUG   HandlerThread:2115858 [system_info.py:_save_code():67] Saving code done
+2025-01-06 20:50:04,506 DEBUG   HandlerThread:2115858 [system_info.py:_save_patches():84] Saving git patches
+2025-01-06 20:50:04,573 DEBUG   HandlerThread:2115858 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-06 20:50:04,573 INFO    HandlerThread:2115858 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-06 20:50:04,576 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:50:04,576 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-06 20:50:04,577 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-06 20:50:04,577 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-06 20:50:04,817 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/uvzr5k1n-wandb-metadata.json
+2025-01-06 20:50:04,822 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-06 20:50:04,822 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: python_packages
+2025-01-06 20:50:04,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:50:04,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:50:04,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:04,825 INFO    HandlerThread:2115858 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/tb"
+  save: true
+}
+control {
+  relay_id: "25145b14587e4479b2cc6a16bb29b0b0"
+}
+_info {
+  stream_id: "bkletf5j"
+}
+
+2025-01-06 20:50:04,826 DEBUG   HandlerThread:2115858 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-06 20:50:04,874 INFO    wandb-upload_1:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/eyhmgilq-code/FISH/eval_robot.py
+2025-01-06 20:50:04,875 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/tq8kyakw-diff.patch
+2025-01-06 20:50:04,978 DEBUG   SenderThread:2115858 [sender.py:send():379] send: telemetry
+2025-01-06 20:50:04,978 DEBUG   SenderThread:2115858 [sender.py:send():379] send: tbrecord
+2025-01-06 20:50:04,979 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:50:04,979 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0 with policy live
+2025-01-06 20:50:05,178 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/5jzgbk5c-events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:50:05,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:05,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/diff.patch
+2025-01-06 20:50:05,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code/FISH/eval_robot.py
+2025-01-06 20:50:05,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:50:05,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/requirements.txt
+2025-01-06 20:50:05,389 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-metadata.json
+2025-01-06 20:50:05,389 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code
+2025-01-06 20:50:05,389 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code/FISH
+2025-01-06 20:50:05,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:06,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:07,387 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:07,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:08,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:09,543 DEBUG   SenderThread:2115858 [sender.py:send():379] send: config
+2025-01-06 20:50:09,543 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:09,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:10,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:11,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:11,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:12,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:13,389 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:13,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:14,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:14,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:15,390 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:15,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:16,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:17,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:18,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:19,391 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:19,822 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:50:19,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:50:19,865 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:19,990 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:20,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:21,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:22,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:23,392 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:23,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:24,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:25,392 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:25,643 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:25,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:26,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:27,393 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:27,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:28,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:29,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:30,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:30,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:31,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:32,833 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:33,833 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:34,822 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:50:34,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:50:34,869 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:35,833 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:35,919 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:36,395 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml
+2025-01-06 20:50:36,834 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:37,834 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:38,838 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:39,838 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:40,838 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:41,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:41,839 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:42,839 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:43,840 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:44,840 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:45,840 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:46,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:46,840 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:47,840 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:48,841 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:49,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:50:49,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:50:49,869 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:50,841 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:51,842 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:51,916 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:52,844 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:53,631 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:50:53,631 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4 with policy now
+2025-01-06 20:50:53,631 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:50:53,632 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:50:53,633 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:50:53,634 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:50:53,844 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:53,910 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/fnlhfe46-media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4
+2025-01-06 20:50:54,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:50:54,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4
+2025-01-06 20:50:54,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media
+2025-01-06 20:50:54,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos
+2025-01-06 20:50:54,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval
+2025-01-06 20:50:54,706 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/kx14jzv7-media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4
+2025-01-06 20:50:54,845 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:55,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:55,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:56,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:57,245 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:57,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:58,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:59,402 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:59,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:00,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:01,403 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:01,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:02,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:02,998 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:03,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:04,487 DEBUG   SystemMonitor:2115858 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-06 20:51:04,488 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:51:04,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:51:04,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:51:04,865 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:05,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:06,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:07,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:08,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:08,923 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:09,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:10,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:11,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:12,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:13,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:14,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:14,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:15,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:16,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:17,408 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:51:17,650 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/8q9iiag4-events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:51:17,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:18,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:19,409 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:19,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:51:19,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:51:19,865 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:19,890 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:20,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:21,409 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:21,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:22,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:23,410 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:23,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:24,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:25,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:25,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:26,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:27,850 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:28,850 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:29,850 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:30,850 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:31,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:31,850 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:32,851 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:33,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:34,488 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:51:34,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:51:34,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:51:34,851 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:35,851 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:36,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:37,414 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:37,645 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:37,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:38,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:39,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:40,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:41,416 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:41,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:42,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:43,059 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:43,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:44,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:45,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:46,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:47,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:48,135 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:48,748 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:51:48,748 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4 with policy now
+2025-01-06 20:51:48,752 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:51:48,753 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:51:48,753 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:51:48,754 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:51:48,856 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:49,017 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/0l73ku16-media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4
+2025-01-06 20:51:49,419 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:51:49,419 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:49,419 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4
+2025-01-06 20:51:49,419 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval
+2025-01-06 20:51:49,703 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/794wq0tz-media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4
+2025-01-06 20:51:49,830 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:51:49,831 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:51:49,873 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:50,857 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:51,419 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:51,858 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:52,857 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:53,420 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:53,629 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:53,858 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:54,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:55,421 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:55,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:56,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:57,421 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:57,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:58,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:59,313 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:51:59,314 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:51:59,314 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:59,314 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:51:59,422 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:51:59,422 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:59,422 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:51:59,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:00,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:01,423 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:01,860 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:02,860 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:03,424 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:03,860 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:04,489 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:52:04,489 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:04,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:52:04,826 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:52:04,873 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:05,860 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:06,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:07,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:08,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:09,637 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:09,881 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:10,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:11,426 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:11,897 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:12,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:13,427 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:13,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:14,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:14,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:15,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:16,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:17,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:18,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:19,429 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:19,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:52:19,826 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:52:19,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:19,921 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:20,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:21,430 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:21,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:22,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:23,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:24,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:25,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:25,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:26,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:27,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:28,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:29,864 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:30,601 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:52:30,601 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4 with policy now
+2025-01-06 20:52:30,602 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:52:30,603 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:52:30,604 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:52:30,605 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:52:30,868 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:30,877 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/13jawk4c-media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4
+2025-01-06 20:52:31,433 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:52:31,434 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:31,434 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4
+2025-01-06 20:52:31,434 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval
+2025-01-06 20:52:31,629 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:31,737 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/poj8jw73-media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4
+2025-01-06 20:52:31,867 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:32,867 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:33,434 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:33,867 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:34,489 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:52:34,830 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:52:34,830 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:52:34,868 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:35,435 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:35,870 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:36,872 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:36,914 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:37,871 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:38,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:39,436 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:39,873 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:40,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:41,065 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:52:41,066 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:52:41,066 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:52:41,437 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:52:41,437 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:52:41,680 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/gmoyno9l-events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:52:41,872 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:42,098 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:42,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:43,438 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:43,880 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:44,877 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:45,873 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:46,873 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:47,128 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:47,439 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:47,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:48,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:49,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:52:49,827 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:52:49,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:50,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:51,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:52,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:52,933 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:53,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:54,875 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:55,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:56,875 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:57,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:58,829 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:58,875 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:59,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:00,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:01,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:02,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:03,829 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:03,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:04,489 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:53:04,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:53:04,827 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:53:04,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:05,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:06,877 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:07,877 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:08,877 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:08,908 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:09,877 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:10,878 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:11,878 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:12,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:13,193 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:53:13,194 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:53:13,194 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4 with policy now
+2025-01-06 20:53:13,194 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:53:13,194 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:53:13,194 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:53:13,449 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:53:13,449 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:13,450 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4
+2025-01-06 20:53:13,450 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval
+2025-01-06 20:53:13,465 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/sbu4sowi-media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4
+2025-01-06 20:53:13,745 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/9g33yy8w-media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4
+2025-01-06 20:53:13,890 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:14,212 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:14,896 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:15,450 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:15,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:16,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:17,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:18,885 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:19,451 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:19,617 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:19,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:53:19,827 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:53:19,881 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:20,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:21,452 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:21,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:22,886 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:23,121 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:53:23,121 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:53:23,122 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:53:23,453 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:53:23,453 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:53:23,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:24,838 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:24,886 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:25,454 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:25,886 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:26,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:27,890 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:28,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:29,455 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:29,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:30,831 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:30,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:31,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:32,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:33,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:34,490 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:53:34,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:53:34,827 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:53:34,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:35,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:35,901 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:36,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:37,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:38,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:39,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:40,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:41,832 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:41,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:42,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:43,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:44,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:45,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:46,832 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:46,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:47,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:48,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:49,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:53:49,827 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:53:49,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:50,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:51,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:51,945 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:52,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:53,890 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:54,890 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:55,892 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:56,895 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:56,949 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:56,953 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:53:56,953 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4 with policy now
+2025-01-06 20:53:56,956 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:53:56,956 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:53:56,957 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:53:56,958 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:53:57,225 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/e632bi05-media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4
+2025-01-06 20:53:57,465 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:53:57,465 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:57,465 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4
+2025-01-06 20:53:57,465 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval
+2025-01-06 20:53:57,759 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/fn7avwrs-media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4
+2025-01-06 20:53:57,903 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:58,897 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:59,465 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:59,900 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:00,913 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:01,466 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:01,900 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:02,733 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:54:02,908 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:03,919 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:04,490 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:54:04,841 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:54:04,841 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:54:04,916 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:05,467 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:05,917 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:06,174 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:54:06,174 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:06,174 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:06,216 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 20:54:06,216 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 20:54:06,380 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table
+2025-01-06 20:54:06,701 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/t9sie1ly-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-06 20:54:06,714 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/qnpv54y7-events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:54:06,729 INFO    wandb-upload_1:2115858 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppyoaom4f
+2025-01-06 20:54:06,791 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 20:54:06,800 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:54:06,909 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:07,141 INFO    SenderThread:2115858 [sender.py:send_request_log_artifact():1525] logged artifact run-bkletf5j-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM0MTgxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY1Mzc2Mg==', 'latestArtifact': None}}
+2025-01-06 20:54:07,141 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:54:07,141 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2025-01-06 20:54:07,141 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:54:07,142 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,143 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,143 DEBUG   SenderThread:2115858 [sender.py:send():379] send: telemetry
+2025-01-06 20:54:07,143 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,143 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,143 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,144 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,144 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,144 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,144 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,146 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,146 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,146 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,146 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,147 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,147 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,147 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,147 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,148 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,148 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,148 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,148 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,148 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,149 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,149 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,149 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 20:54:07,376 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/0n8y3ctg-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-06 20:54:07,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:07,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:07,469 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:54:07,469 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json
+2025-01-06 20:54:07,469 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval
+2025-01-06 20:54:07,522 INFO    wandb-upload_1:2115858 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprhug1a3e
+2025-01-06 20:54:07,650 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/fkyioxav-media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json
+2025-01-06 20:54:07,909 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:07,963 INFO    SenderThread:2115858 [sender.py:send_request_log_artifact():1525] logged artifact run-bkletf5j-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM0MTg0OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY1Mzc2NQ==', 'latestArtifact': None}}
+2025-01-06 20:54:07,963 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:54:07,963 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:54:07,964 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json with policy now
+2025-01-06 20:54:07,964 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:54:07,964 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,964 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:08,210 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/dxsdmipb-media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json
+2025-01-06 20:54:08,469 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:08,909 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:09,469 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:09,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:10,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:11,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:12,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:13,835 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:54:13,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:14,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:15,725 DEBUG   SenderThread:2115858 [sender.py:send():379] send: exit
+2025-01-06 20:54:15,725 INFO    SenderThread:2115858 [sender.py:send_exit():586] handling exit code: 0
+2025-01-06 20:54:15,726 INFO    SenderThread:2115858 [sender.py:send_exit():588] handling runtime: 251
+2025-01-06 20:54:15,726 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:15,726 INFO    SenderThread:2115858 [sender.py:send_exit():594] send defer
+2025-01-06 20:54:15,726 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:15,726 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-06 20:54:15,726 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:15,726 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-06 20:54:15,726 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 1
+2025-01-06 20:54:15,726 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:15,727 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-06 20:54:15,727 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:15,727 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-06 20:54:15,727 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 2
+2025-01-06 20:54:15,727 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:15,727 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-06 20:54:15,727 INFO    HandlerThread:2115858 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-06 20:54:15,727 DEBUG   SystemMonitor:2115858 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-06 20:54:15,727 DEBUG   SystemMonitor:2115858 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-06 20:54:15,728 INFO    HandlerThread:2115858 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-06 20:54:15,728 INFO    HandlerThread:2115858 [interfaces.py:finish():200] Joined disk monitor
+2025-01-06 20:54:15,781 INFO    HandlerThread:2115858 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-06 20:54:15,781 INFO    HandlerThread:2115858 [interfaces.py:finish():200] Joined memory monitor
+2025-01-06 20:54:15,781 INFO    HandlerThread:2115858 [interfaces.py:finish():200] Joined network monitor
+2025-01-06 20:54:15,781 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:15,781 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-06 20:54:15,781 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 3
+2025-01-06 20:54:15,782 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:54:15,782 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:15,782 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-06 20:54:15,782 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:15,782 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-06 20:54:15,782 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 4
+2025-01-06 20:54:15,782 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:15,782 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-06 20:54:16,472 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:17,472 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:22,179 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 20:54:22,179 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:54:22,179 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-06 20:54:22,180 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:22,180 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-06 20:54:22,180 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 5
+2025-01-06 20:54:22,180 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 20:54:22,180 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:54:22,180 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:22,181 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:22,181 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-06 20:54:22,181 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:22,182 DEBUG   SenderThread:2115858 [sender.py:send():379] send: summary
+2025-01-06 20:54:22,185 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:22,186 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:22,186 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-06 20:54:22,186 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 6
+2025-01-06 20:54:22,186 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:22,186 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-06 20:54:22,186 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:22,186 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-06 20:54:22,190 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:54:22,301 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 7
+2025-01-06 20:54:22,302 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:22,302 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-06 20:54:22,302 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:22,302 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-06 20:54:22,475 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:22,475 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml
+2025-01-06 20:54:22,727 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 20:54:24,861 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 8
+2025-01-06 20:54:24,861 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 20:54:24,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:24,861 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-06 20:54:24,862 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:24,862 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-06 20:54:24,862 INFO    SenderThread:2115858 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-06 20:54:24,864 INFO    SenderThread:2115858 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-06 20:54:24,866 INFO    SenderThread:2115858 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-06 20:54:24,881 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 9
+2025-01-06 20:54:24,881 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:24,881 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-06 20:54:24,881 DEBUG   SenderThread:2115858 [sender.py:send():379] send: artifact
+2025-01-06 20:54:25,113 INFO    SenderThread:2115858 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}}
+2025-01-06 20:54:25,113 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:25,113 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-06 20:54:25,113 INFO    SenderThread:2115858 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml config.yaml
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json wandb-summary.json
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0 events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/requirements.txt requirements.txt
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-metadata.json wandb-metadata.json
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log output.log
+2025-01-06 20:54:25,477 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/diff.patch diff.patch
+2025-01-06 20:54:25,480 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-06 20:54:25,482 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4 media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4
+2025-01-06 20:54:25,483 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4 media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4 media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4 media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4 media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 10
+2025-01-06 20:54:25,485 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:25,485 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-06 20:54:25,486 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:25,486 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-06 20:54:25,486 INFO    SenderThread:2115858 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 20:54:25,663 INFO    wandb-upload_1:2115858 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:25,668 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml
+2025-01-06 20:54:25,705 INFO    wandb-upload_3:2115858 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/requirements.txt
+2025-01-06 20:54:25,719 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/sacbiy0i-events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:54:25,728 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 20:54:25,728 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 20:54:25,769 INFO    wandb-upload_4:2115858 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:25,969 INFO    Thread-11 (_thread_body):2115858 [sender.py:transition_state():614] send defer: 11
+2025-01-06 20:54:25,969 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:25,969 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-06 20:54:25,969 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:25,970 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-06 20:54:25,970 INFO    SenderThread:2115858 [file_pusher.py:join():175] waiting for file pusher
+2025-01-06 20:54:25,971 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 12
+2025-01-06 20:54:25,971 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:25,971 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-06 20:54:25,971 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:25,971 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-06 20:54:25,971 INFO    SenderThread:2115858 [file_stream.py:finish():601] file stream finish called
+2025-01-06 20:54:26,207 INFO    SenderThread:2115858 [file_stream.py:finish():605] file stream finish is done
+2025-01-06 20:54:26,207 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 13
+2025-01-06 20:54:26,207 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:26,207 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-06 20:54:26,207 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:26,207 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-06 20:54:26,207 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 14
+2025-01-06 20:54:26,207 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:26,207 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-06 20:54:26,208 DEBUG   SenderThread:2115858 [sender.py:send():379] send: final
+2025-01-06 20:54:26,208 DEBUG   SenderThread:2115858 [sender.py:send():379] send: footer
+2025-01-06 20:54:26,208 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:26,208 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-06 20:54:26,208 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 20:54:26,208 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 20:54:26,209 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: server_info
+2025-01-06 20:54:26,209 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: server_info
+2025-01-06 20:54:26,210 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 20:54:26,210 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-06 20:54:26,211 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-06 20:54:26,211 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:26,262 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 20:54:26,262 INFO    MainThread:2115858 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-06 20:54:26,263 INFO    MainThread:2115858 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-06 20:54:26,263 INFO    MainThread:2115858 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-06 20:54:26,263 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-06 20:54:26,263 INFO    HandlerThread:2115858 [handler.py:finish():882] shutting down handler
+2025-01-06 20:54:27,210 INFO    WriterThread:2115858 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/run-bkletf5j.wandb
+2025-01-06 20:54:27,262 INFO    SenderThread:2115858 [sender.py:finish():1615] shutting down sender
+2025-01-06 20:54:27,262 INFO    SenderThread:2115858 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 20:54:27,262 INFO    SenderThread:2115858 [file_pusher.py:join():175] waiting for file pusher
diff --git a/bkletf5j/wandb/debug.log b/bkletf5j/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..09ef718cf064dc90d0ff3e0f9084528f7c2137a2
--- /dev/null
+++ b/bkletf5j/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Configure stats pid to 2115685
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/settings
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/logs/debug.log
+2025-01-06 20:50:04,129 INFO    MainThread:2115685 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/logs/debug-internal.log
+2025-01-06 20:50:04,129 INFO    MainThread:2115685 [wandb_init.py:init():569] calling init triggers
+2025-01-06 20:50:04,129 INFO    MainThread:2115685 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j'}
+2025-01-06 20:50:04,129 INFO    MainThread:2115685 [wandb_init.py:init():619] starting backend
+2025-01-06 20:50:04,129 INFO    MainThread:2115685 [wandb_init.py:init():623] setting up manager
+2025-01-06 20:50:04,133 INFO    MainThread:2115685 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-06 20:50:04,134 INFO    MainThread:2115685 [wandb_init.py:init():631] backend started and connected
+2025-01-06 20:50:04,146 INFO    MainThread:2115685 [wandb_init.py:init():720] updated telemetry
+2025-01-06 20:50:04,153 INFO    MainThread:2115685 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-06 20:50:04,401 INFO    MainThread:2115685 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-06 20:50:04,470 INFO    MainThread:2115685 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-06 20:50:04,470 INFO    MainThread:2115685 [wandb_init.py:init():804] starting run threads in backend
+2025-01-06 20:50:04,823 INFO    MainThread:2115685 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-06 20:50:04,823 INFO    MainThread:2115685 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-06 20:50:04,823 INFO    MainThread:2115685 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-06 20:50:04,823 INFO    MainThread:2115685 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-06 20:50:04,825 INFO    MainThread:2115685 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-06 20:50:04,825 INFO    MainThread:2115685 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/tb, True
+2025-01-06 20:50:09,543 INFO    MainThread:2115685 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'}
+2025-01-06 20:54:27,264 WARNING MsgRouterThr:2115685 [router.py:message_loop():77] message_loop has been closed
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code/FISH/eval_robot.py b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9f726c2d982daa075a1ad737b1590515d1a00bd4
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1002_0'
+wandb_notes:
+  desc: null
+  value: 1002_0_req_1365_0
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 5
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1736214604
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: fowlers
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/diff.patch b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..e24d60575094ab086c08b663f56c41b1223113ec
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/diff.patch
@@ -0,0 +1,65 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index d2afe29..3e5ed57 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -133,12 +133,19 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
+-# wandb_run_id: '1002_0' 
++wandb_run_id: '1002_0' 
++# wandb_run_id: '1009_1' # seed 1
++# wandb_run_id: '1022_0' # dataset shuffle seed 1
++
++# RGBD+mask+contact(all ftrs)+act history
++# wandb_run_id: '1003_0' 
++# wandb_run_id: '1013_1' # seed 1
++# wandb_run_id: '1017_0' # dataset shuffle seed 1
+ 
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1007_0' 
+-wandb_run_id: '1922_1' # seed 1
++# wandb_run_id: '1922_1' # seed 1
+ # wandb_run_id: '1938_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1
+ # wandb_run_id: '1926_1' # seed 1
+ # wandb_run_id: '1930_0' # dataset shuffle seed 1
+ 
+-
+-
+ # RGBD+mask+contact(contact+DTC)+act history
+ # wandb_run_id: '1745_0'
+ 
++# all books, 4/20 demos per book
++
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+ 
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index c7eadce..e3c7dc1 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -161,8 +161,18 @@ run_id_list = [
+ run_id_list = [
+     # '1922_1',
+     # '1922_2',
+-    '1926_1',
+-    '1930_0',
++    # '1944_0',
++    # '1972_0',
++    # '1976_1',
++    # '1948_0',
++    # '1950_0',
++    # '1954_1',
++    '1002_0',
++    '1009_1',
++    '1022_0',
++    '1003_0',
++    '1013_1',
++    '1017_0',
+ ]
+ 
+ checkpoint_epoch = 12000
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]}
\ No newline at end of file
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..d036dd6e4aa25700d7298da4ac18f480e25c03e7
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[11, 20, 0.55, 0.29075199369800797, 0.7880825101753088, 0.342133939297778, 0.7678436487377445, 0.37386482956547973, 0.7163380607529932, 0.39258463310363445, 0.7027052752268215]]}
\ No newline at end of file
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4 b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..0b9fe80e3257d592d3ff3d0d7f162abad2cdaa9b
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5629c4a8c29fc9b2a6bcd5b52045b248fa99a988b49da528172799d54ae9e43
+size 1138009
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4 b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..dfe1619b871855b91927fe6b690d51ff298b0bea
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0cb5720b5b83600f05130a6d5d7c655d93900882d84fc0a0fcb4fc77ed3ee7a
+size 1135045
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4 b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..581bcb00885fab23c514e284e49d008921d20b6d
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cf0e0208d2fc5ae593796264b65c54bc47568501b27870d0019ac790e381dd1
+size 1137418
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4 b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1cc1459b21506352e313e09f574fccd7e87883fa
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f993ed9f4bf0eae5cf19f3476f8fdd87803f51f4afc6fb30ec1d84255a3e003
+size 1130405
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4 b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..44e45bfc1b7a85fe35c98e20a7ceb6efe9f351db
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a8935b2358637140fbc934be6af895927356f6549f7c53e6b407ab74efdc608
+size 1186142
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..a7d5e3d84a33e28e794d3ecee2c7d2420446954c
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
@@ -0,0 +1,283 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1736214609.543816]: resetting environment
+[INFO] [1736214609.549294]: cleared current plan
+[INFO] [1736214609.550124]: moving to home
+[INFO] [1736214610.551956]: reached home
+[INFO] [1736214610.553236]: reset action history
+[INFO] [1736214613.262226]: environment reset
+Starting episode 0
+[INFO] [1736214613.264021]: resetting environment
+[INFO] [1736214613.267083]: cleared current plan
+[INFO] [1736214613.268161]: moving to home
+[INFO] [1736214614.271151]: reached home
+[INFO] [1736214614.274329]: reset action history
+[INFO] [1736214616.982799]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1736214621.926328]: resetting environment
+[INFO] [1736214621.929699]: cleared current plan
+[INFO] [1736214621.930749]: moving to home
+[INFO] [1736214622.932518]: reached home
+[INFO] [1736214622.933672]: reset action history
+[INFO] [1736214625.642555]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/episode_rosbags/episode_0_2025-01-06-20-50-25.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2115685.log
+started roslaunch server http://158.130.50.37:43103/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2115971]
+process[print_text-2]: started with pid [2115995]
+process[rosbag_record-3]: started with pid [2115996]
+started bagging!
+[INFO] [1736214652.651797]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit[pub_text-1] killing on exit
+[INFO] [1736214653.402180]: Stored episode 1.
+[INFO] [1736214653.402425]: Saving video...
+[INFO] [1736214653.618484]: Video saved!
+Starting episode 1
+[INFO] [1736214653.634687]: resetting environment
+[INFO] [1736214653.638837]: cleared current plan
+[INFO] [1736214653.639869]: moving to home
+[INFO] [1736214657.245008]: reached home
+[INFO] [1736214657.246209]: reset action history
+[INFO] [1736214659.953606]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736214676.701494]: resetting environment
+[INFO] [1736214676.703818]: cleared current plan
+[INFO] [1736214676.703984]: moving to home
+[INFO] [1736214677.705241]: reached home
+[INFO] [1736214677.706033]: reset action history
+[INFO] [1736214680.412853]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/episode_rosbags/episode_1_2025-01-06-20-51-20.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2115685.log
+started roslaunch server http://158.130.50.37:40247/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2116108]
+process[print_text-5]: started with pid [2116109]
+process[rosbag_record-6]: started with pid [2116133]
+started bagging!
+For topic gripper_width: timestamp difference is 102777635 for nearest: 1736214694599895870 - target: 1736214694497118235 at idx 191
+For topic gripper_width: timestamp difference is 103166449 for nearest: 1736214694799894202 - target: 1736214694696727753 at idx 193
+For topic gripper_width: timestamp difference is 104797765 for nearest: 1736214695599894449 - target: 1736214695495096684 at idx 204
+For topic gripper_width: timestamp difference is 113245533 for nearest: 1736214699799894617 - target: 1736214699686649084 at idx 249
+For topic gripper_width: timestamp difference is 113673163 for nearest: 1736214699999919606 - target: 1736214699886246443 at idx 249
+[INFO] [1736214707.370004]: Storing episode...
+[rosbag_record-6] killing on exit
+[pub_text-4] killing on exit
+[print_text-5] killing on exit
+[INFO] [1736214708.135141]: Stored episode 2.
+[INFO] [1736214708.135627]: Saving video...
+[WARN] [1736214708.709786]: Plan exhausted
+[WARN] [1736214708.748744]: Plan exhausted
+[INFO] [1736214708.711211]: Video saved!
+Starting episode 2
+[INFO] [1736214708.755467]: resetting environment
+[INFO] [1736214708.769162]: cleared current plan
+[INFO] [1736214708.773032]: moving to home
+[INFO] [1736214712.581875]: reached home
+[INFO] [1736214712.587604]: reset action history
+[INFO] [1736214715.317071]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736214718.540256]: resetting environment
+[INFO] [1736214718.546123]: cleared current plan
+[INFO] [1736214718.546306]: moving to home
+[INFO] [1736214719.547737]: reached home
+[INFO] [1736214719.548152]: reset action history
+[INFO] [1736214722.255928]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/episode_rosbags/episode_2_2025-01-06-20-52-02.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2115685.log
+started roslaunch server http://158.130.50.37:44493/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2116244]
+process[print_text-8]: started with pid [2116245]
+process[rosbag_record-9]: started with pid [2116246]
+started bagging!
+For topic gripper_width: timestamp difference is 102394939 for nearest: 1736214729633223892 - target: 1736214729530828953 at idx 116
+For topic gripper_width: timestamp difference is 102586690 for nearest: 1736214729833258334 - target: 1736214729730671644 at idx 117
+For topic gripper_width: timestamp difference is 105361941 for nearest: 1736214731834424021 - target: 1736214731729062080 at idx 144
+For topic gripper_width: timestamp difference is -223543510 for nearest: 1736214738333252325 - target: 1736214738556795835 at idx 239
+For topic gripper_width: timestamp difference is 110045180 for nearest: 1736214739033227667 - target: 1736214738923182487 at idx 244
+For topic gripper_width: timestamp difference is -122808570 for nearest: 1736214739433220988 - target: 1736214739556029558 at idx 247
+[INFO] [1736214749.225302]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1736214749.989322]: Stored episode 3.
+[INFO] [1736214749.990314]: Saving video...
+[INFO] [1736214750.563512]: Video saved!
+Starting episode 3
+[INFO] [1736214750.604374]: resetting environment
+[INFO] [1736214750.622622]: cleared current plan
+[INFO] [1736214750.626523]: moving to home
+[WARN] [1736214750.549451]: Plan exhausted
+[WARN] [1736214750.589687]: Plan exhausted
+[INFO] [1736214754.434608]: reached home
+[INFO] [1736214754.439811]: reset action history
+[INFO] [1736214757.356144]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736214761.075297]: resetting environment
+[INFO] [1736214761.088187]: cleared current plan
+[INFO] [1736214761.092354]: moving to home
+[INFO] [1736214762.101256]: reached home
+[INFO] [1736214762.106191]: reset action history
+[INFO] [1736214764.882424]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/episode_rosbags/episode_3_2025-01-06-20-52-44.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2115685.log
+started roslaunch server http://158.130.50.37:38547/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2116381]
+process[print_text-11]: started with pid [2116382]
+process[rosbag_record-12]: started with pid [2116406]
+started bagging!
+[INFO] [1736214791.835982]: Storing episode...
+[rosbag_record-12] killing on exit
+[pub_text-10] killing on exit
+[print_text-11] killing on exit
+[INFO] [1736214792.632168]: Stored episode 4.
+[INFO] [1736214792.632711]: Saving video...
+[INFO] [1736214793.146263]: Video saved!
+Starting episode 4
+[INFO] [1736214793.185732]: resetting environment
+[INFO] [1736214793.205585]: cleared current plan
+[INFO] [1736214793.211404]: moving to home
+[INFO] [1736214796.823836]: reached home
+[INFO] [1736214796.830822]: reset action history
+[INFO] [1736214799.613740]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736214802.970658]: resetting environment
+[INFO] [1736214802.989327]: cleared current plan
+[INFO] [1736214802.989673]: moving to home
+[INFO] [1736214803.993293]: reached home
+[INFO] [1736214803.996832]: reset action history
+[INFO] [1736214806.795654]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/episode_rosbags/episode_4_2025-01-06-20-53-26.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2115685.log
+started roslaunch server http://158.130.50.37:35283/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2116537]
+process[print_text-14]: started with pid [2116538]
+process[rosbag_record-15]: started with pid [2116539]
+started bagging!
+[INFO] [1736214835.363155]: Storing episode...
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit
+[print_text-14] killing on exit
+[INFO] [1736214836.189115]: Stored episode 5.
+[INFO] [1736214836.197029]: Saving video...
+[WARN] [1736214836.697966]: Plan exhausted
+[WARN] [1736214836.733400]: Plan exhausted
+[WARN] [1736214836.769312]: Plan exhausted
+[WARN] [1736214836.815802]: Plan exhausted
+[WARN] [1736214836.853665]: Plan exhausted
+[WARN] [1736214836.889614]: Plan exhausted
+[WARN] [1736214836.929493]: Plan exhausted
+[WARN] [1736214836.969481]: Plan exhausted
+[INFO] [1736214836.904244]: Video saved!
+[INFO] [1736214836.958393]: resetting environment
+[INFO] [1736214836.973395]: cleared current plan
+[INFO] [1736214836.982543]: moving to home
+[INFO] [1736214840.691463]: reached home
+[INFO] [1736214840.700700]: reset action history
+[INFO] [1736214843.501756]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/requirements.txt b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-metadata.json b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..685207ccb840cc66267c16b594fffe5112999719
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-07T01:50:04.498228",
+    "startedAt": "2025-01-07T01:50:04.126209",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3561.57975,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4495.894,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 525.3030891418457
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..78956cbcd6c70bf8994cba2be19a56714eadd4d3
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "d5629c4a8c29fc9b2a6bcd5b52045b248fa99a988b49da528172799d54ae9e43", "size": 1138009, "path": "media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4"}, "global_step": 665, "_timestamp": 1736214846.7058787, "_runtime": 242.57108664512634, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "b0cb5720b5b83600f05130a6d5d7c655d93900882d84fc0a0fcb4fc77ed3ee7a", "size": 1135045, "path": "media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4"}, "eval/num_success": 5.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "4cf0e0208d2fc5ae593796264b65c54bc47568501b27870d0019ac790e381dd1", "size": 1137418, "path": "media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "8f993ed9f4bf0eae5cf19f3476f8fdd87803f51f4afc6fb30ec1d84255a3e003", "size": 1130405, "path": "media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "6a8935b2358637140fbc934be6af895927356f6549f7c53e6b407ab74efdc608", "size": 1186142, "path": "media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://07kjmesd9x38ye7n0okrwdikvlbm1vqfda2s4vskx92mjtjwk77k5j7koesdgju8qvq1y57m87qksxvqs23aq2l77zny7nrgcl3jistf1bki9xju0mknpe79wck9jdhi/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://f4f6n9lsqxwn495kysr3be4v7xlodgak79rdjny8g2kh50p68zgvq2tqh9pu6ml4kv7x1aagi04x866avfh1wsxnpxiusf3g1d4e4btirejar7d1lbq093zn4cw3wdi9:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 11, "total_num_episodes": 20, "total_success_rate": 0.55, "total_success_rate_lb_0.01": 0.29075199369800797, "total_success_rate_ub_0.01": 0.7880825101753088, "total_success_rate_lb_0.025": 0.342133939297778, "total_success_rate_ub_0.025": 0.7678436487377445, "total_success_rate_lb_0.05": 0.37386482956547973, "total_success_rate_ub_0.05": 0.7163380607529932, "total_success_rate_lb_0.1": 0.39258463310363445, "total_success_rate_ub_0.1": 0.7027052752268215, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "4142c490068220aa55977f82e69ae7294a368cb4ff2bbf9c1dd35a4c0dff74b0", "size": 508, "artifact_path": "wandb-client-artifact://ywkghs2rlig6wxl4el8z1dui5mwm1ka4tt2tj5rgxapapvgf1hj4p8o4jckacts8etiqzclq4ybf8o64hwk5ene3d76gmu108bnqxjnilnqclsatk30mjbqtwthqi5e8/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vyrf58fjrrjk07vj21d45z2yqycsl7m1a4qu01uu6fu2u3zjwfeg77wk2zftm0o6elzx14cs6vc95t3g0wtkn43ht40dgw9jdufa018vwvayg7oei7lir4ueitp4weko:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 251}, "eval/total_success_rate": 0.550000011920929, "eval/total_success_rate_lb_0.01": 0.29075199365615845, "eval/total_success_rate_ub_0.01": 0.788082480430603, "eval/total_success_rate_lb_0.025": 0.34213393926620483, "eval/total_success_rate_ub_0.025": 0.7678436636924744, "eval/total_success_rate_lb_0.05": 0.3738648295402527, "eval/total_success_rate_ub_0.05": 0.716338038444519, "eval/total_success_rate_lb_0.1": 0.3925846219062805, "eval/total_success_rate_ub_0.1": 0.7027052640914917}
\ No newline at end of file
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/logs/debug-internal.log b/bkletf5j/wandb/run-20250106_205004-bkletf5j/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..58f6bf74ea362f4b0ee338d3997c049de51b3430
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/logs/debug-internal.log
@@ -0,0 +1,748 @@
+2025-01-06 20:50:04,134 INFO    StreamThr :2115858 [internal.py:wandb_internal():85] W&B internal server running at pid: 2115858, started at: 2025-01-06 20:50:04.134315
+2025-01-06 20:50:04,135 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status
+2025-01-06 20:50:04,146 INFO    WriterThread:2115858 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/run-bkletf5j.wandb
+2025-01-06 20:50:04,147 DEBUG   SenderThread:2115858 [sender.py:send():379] send: header
+2025-01-06 20:50:04,155 DEBUG   SenderThread:2115858 [sender.py:send():379] send: run
+2025-01-06 20:50:04,386 INFO    SenderThread:2115858 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files
+2025-01-06 20:50:04,386 INFO    SenderThread:2115858 [sender.py:_start_run_threads():1188] run started: bkletf5j with start time 1736214604.134792
+2025-01-06 20:50:04,401 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: check_version
+2025-01-06 20:50:04,401 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: check_version
+2025-01-06 20:50:04,475 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: run_start
+2025-01-06 20:50:04,483 DEBUG   HandlerThread:2115858 [system_info.py:__init__():26] System info init
+2025-01-06 20:50:04,483 DEBUG   HandlerThread:2115858 [system_info.py:__init__():41] System info init done
+2025-01-06 20:50:04,483 INFO    HandlerThread:2115858 [system_monitor.py:start():194] Starting system monitor
+2025-01-06 20:50:04,483 INFO    SystemMonitor:2115858 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-06 20:50:04,483 INFO    HandlerThread:2115858 [system_monitor.py:probe():214] Collecting system info
+2025-01-06 20:50:04,485 INFO    SystemMonitor:2115858 [interfaces.py:start():188] Started cpu monitoring
+2025-01-06 20:50:04,485 INFO    SystemMonitor:2115858 [interfaces.py:start():188] Started disk monitoring
+2025-01-06 20:50:04,486 INFO    SystemMonitor:2115858 [interfaces.py:start():188] Started gpu monitoring
+2025-01-06 20:50:04,487 INFO    SystemMonitor:2115858 [interfaces.py:start():188] Started memory monitoring
+2025-01-06 20:50:04,487 INFO    SystemMonitor:2115858 [interfaces.py:start():188] Started network monitoring
+2025-01-06 20:50:04,498 DEBUG   HandlerThread:2115858 [system_info.py:probe():152] Probing system
+2025-01-06 20:50:04,499 DEBUG   HandlerThread:2115858 [system_info.py:_probe_git():137] Probing git
+2025-01-06 20:50:04,504 DEBUG   HandlerThread:2115858 [system_info.py:_probe_git():145] Probing git done
+2025-01-06 20:50:04,504 DEBUG   HandlerThread:2115858 [system_info.py:probe():200] Probing system done
+2025-01-06 20:50:04,504 DEBUG   HandlerThread:2115858 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:50:04.498228', 'startedAt': '2025-01-07T01:50:04.126209', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.57975, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4495.894, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 525.3030891418457}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-06 20:50:04,504 INFO    HandlerThread:2115858 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-06 20:50:04,504 INFO    HandlerThread:2115858 [system_monitor.py:probe():227] Publishing system info
+2025-01-06 20:50:04,504 DEBUG   HandlerThread:2115858 [system_info.py:_save_code():44] Saving code
+2025-01-06 20:50:04,506 DEBUG   HandlerThread:2115858 [system_info.py:_save_code():67] Saving code done
+2025-01-06 20:50:04,506 DEBUG   HandlerThread:2115858 [system_info.py:_save_patches():84] Saving git patches
+2025-01-06 20:50:04,573 DEBUG   HandlerThread:2115858 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-06 20:50:04,573 INFO    HandlerThread:2115858 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-06 20:50:04,576 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:50:04,576 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-06 20:50:04,577 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-06 20:50:04,577 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-06 20:50:04,817 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/uvzr5k1n-wandb-metadata.json
+2025-01-06 20:50:04,822 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-06 20:50:04,822 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: python_packages
+2025-01-06 20:50:04,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:50:04,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:50:04,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:04,825 INFO    HandlerThread:2115858 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/tb"
+  save: true
+}
+control {
+  relay_id: "25145b14587e4479b2cc6a16bb29b0b0"
+}
+_info {
+  stream_id: "bkletf5j"
+}
+
+2025-01-06 20:50:04,826 DEBUG   HandlerThread:2115858 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-06 20:50:04,874 INFO    wandb-upload_1:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/eyhmgilq-code/FISH/eval_robot.py
+2025-01-06 20:50:04,875 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/tq8kyakw-diff.patch
+2025-01-06 20:50:04,978 DEBUG   SenderThread:2115858 [sender.py:send():379] send: telemetry
+2025-01-06 20:50:04,978 DEBUG   SenderThread:2115858 [sender.py:send():379] send: tbrecord
+2025-01-06 20:50:04,979 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:50:04,979 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0 with policy live
+2025-01-06 20:50:05,178 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/5jzgbk5c-events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:50:05,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:05,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/diff.patch
+2025-01-06 20:50:05,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code/FISH/eval_robot.py
+2025-01-06 20:50:05,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:50:05,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/requirements.txt
+2025-01-06 20:50:05,389 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-metadata.json
+2025-01-06 20:50:05,389 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code
+2025-01-06 20:50:05,389 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code/FISH
+2025-01-06 20:50:05,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:06,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:07,387 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:07,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:08,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:09,543 DEBUG   SenderThread:2115858 [sender.py:send():379] send: config
+2025-01-06 20:50:09,543 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:09,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:10,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:11,388 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:11,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:12,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:13,389 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:13,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:14,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:14,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:15,390 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:15,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:16,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:17,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:18,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:19,391 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:19,822 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:50:19,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:50:19,865 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:19,990 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:20,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:21,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:22,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:23,392 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:23,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:24,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:25,392 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:25,643 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:25,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:26,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:27,393 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:27,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:28,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:29,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:30,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:30,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:31,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:32,833 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:33,833 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:34,822 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:50:34,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:50:34,869 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:35,833 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:35,919 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:36,395 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml
+2025-01-06 20:50:36,834 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:37,834 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:38,838 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:39,838 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:40,838 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:41,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:41,839 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:42,839 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:43,840 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:44,840 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:45,840 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:46,824 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:46,840 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:47,840 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:48,841 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:49,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:50:49,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:50:49,869 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:50,841 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:51,842 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:51,916 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:52,844 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:53,631 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:50:53,631 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4 with policy now
+2025-01-06 20:50:53,631 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:50:53,632 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:50:53,633 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:50:53,634 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:50:53,844 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:53,910 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/fnlhfe46-media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4
+2025-01-06 20:50:54,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:50:54,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4
+2025-01-06 20:50:54,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media
+2025-01-06 20:50:54,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos
+2025-01-06 20:50:54,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval
+2025-01-06 20:50:54,706 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/kx14jzv7-media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4
+2025-01-06 20:50:54,845 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:55,400 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:55,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:56,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:57,245 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:50:57,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:58,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:50:59,402 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:50:59,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:00,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:01,403 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:01,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:02,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:02,998 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:03,846 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:04,487 DEBUG   SystemMonitor:2115858 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-06 20:51:04,488 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:51:04,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:51:04,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:51:04,865 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:05,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:06,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:07,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:08,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:08,923 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:09,847 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:10,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:11,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:12,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:13,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:14,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:14,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:15,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:16,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:17,408 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:51:17,650 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/8q9iiag4-events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:51:17,848 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:18,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:19,409 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:19,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:51:19,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:51:19,865 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:19,890 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:20,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:21,409 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:21,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:22,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:23,410 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:23,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:24,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:25,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:25,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:26,849 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:27,850 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:28,850 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:29,850 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:30,850 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:31,825 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:31,850 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:32,851 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:33,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:34,488 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:51:34,823 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:51:34,823 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:51:34,851 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:35,851 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:36,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:37,414 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:37,645 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:37,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:38,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:39,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:40,852 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:41,416 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:41,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:42,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:43,059 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:43,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:44,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:45,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:46,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:47,853 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:48,135 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:48,748 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:51:48,748 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4 with policy now
+2025-01-06 20:51:48,752 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:51:48,753 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:51:48,753 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:51:48,754 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:51:48,856 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:49,017 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/0l73ku16-media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4
+2025-01-06 20:51:49,419 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:51:49,419 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:49,419 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4
+2025-01-06 20:51:49,419 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval
+2025-01-06 20:51:49,703 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/794wq0tz-media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4
+2025-01-06 20:51:49,830 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:51:49,831 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:51:49,873 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:50,857 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:51,419 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:51,858 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:52,857 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:53,420 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:53,629 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:53,858 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:54,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:55,421 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:55,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:56,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:57,421 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:57,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:58,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:51:59,313 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:51:59,314 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:51:59,314 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:51:59,314 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:51:59,422 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:51:59,422 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:51:59,422 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:51:59,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:00,859 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:01,423 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:01,860 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:02,860 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:03,424 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:03,860 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:04,489 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:52:04,489 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:04,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:52:04,826 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:52:04,873 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:05,860 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:06,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:07,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:08,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:09,637 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:09,881 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:10,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:11,426 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:11,897 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:12,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:13,427 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:13,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:14,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:14,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:15,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:16,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:17,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:18,862 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:19,429 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:19,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:52:19,826 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:52:19,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:19,921 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:20,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:21,430 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:21,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:22,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:23,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:24,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:25,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:25,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:26,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:27,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:28,863 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:29,864 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:30,601 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:52:30,601 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4 with policy now
+2025-01-06 20:52:30,602 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:52:30,603 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:52:30,604 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:52:30,605 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:52:30,868 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:30,877 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/13jawk4c-media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4
+2025-01-06 20:52:31,433 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:52:31,434 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:31,434 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4
+2025-01-06 20:52:31,434 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval
+2025-01-06 20:52:31,629 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:31,737 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/poj8jw73-media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4
+2025-01-06 20:52:31,867 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:32,867 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:33,434 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:33,867 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:34,489 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:52:34,830 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:52:34,830 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:52:34,868 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:35,435 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:35,870 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:36,872 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:36,914 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:37,871 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:38,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:39,436 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:39,873 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:40,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:41,065 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:52:41,066 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:52:41,066 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:52:41,437 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:52:41,437 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:52:41,680 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/gmoyno9l-events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:52:41,872 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:42,098 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:42,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:43,438 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:43,880 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:44,877 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:45,873 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:46,873 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:47,128 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:47,439 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:52:47,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:48,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:49,826 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:52:49,827 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:52:49,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:50,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:51,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:52,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:52,933 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:53,874 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:54,875 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:55,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:56,875 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:57,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:58,829 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:52:58,875 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:52:59,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:00,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:01,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:02,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:03,829 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:03,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:04,489 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:53:04,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:53:04,827 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:53:04,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:05,876 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:06,877 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:07,877 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:08,877 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:08,908 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:09,877 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:10,878 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:11,878 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:12,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:13,193 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:53:13,194 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:53:13,194 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4 with policy now
+2025-01-06 20:53:13,194 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:53:13,194 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:53:13,194 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:53:13,449 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:53:13,449 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:13,450 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4
+2025-01-06 20:53:13,450 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval
+2025-01-06 20:53:13,465 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/sbu4sowi-media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4
+2025-01-06 20:53:13,745 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/9g33yy8w-media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4
+2025-01-06 20:53:13,890 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:14,212 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:14,896 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:15,450 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:15,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:16,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:17,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:18,885 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:19,451 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:19,617 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:19,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:53:19,827 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:53:19,881 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:20,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:21,452 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:21,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:22,886 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:23,121 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:53:23,121 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:53:23,122 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:53:23,453 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:53:23,453 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:53:23,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:24,838 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:24,886 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:25,454 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:25,886 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:26,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:27,890 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:28,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:29,455 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:29,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:30,831 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:30,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:31,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:32,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:33,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:34,490 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:53:34,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:53:34,827 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:53:34,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:35,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:35,901 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:36,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:37,887 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:38,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:39,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:40,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:41,832 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:41,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:42,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:43,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:44,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:45,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:46,832 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:46,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:47,888 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:48,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:49,827 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:53:49,827 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:53:49,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:50,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:51,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:51,945 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:52,889 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:53,890 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:54,890 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:55,892 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:56,895 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:56,949 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:53:56,953 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:53:56,953 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4 with policy now
+2025-01-06 20:53:56,956 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:53:56,956 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:53:56,957 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:53:56,958 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:53:57,225 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/e632bi05-media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4
+2025-01-06 20:53:57,465 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:53:57,465 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:57,465 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4
+2025-01-06 20:53:57,465 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval
+2025-01-06 20:53:57,759 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/fn7avwrs-media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4
+2025-01-06 20:53:57,903 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:58,897 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:53:59,465 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:53:59,900 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:00,913 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:01,466 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:01,900 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:02,733 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:54:02,908 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:03,919 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:04,490 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:54:04,841 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 20:54:04,841 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 20:54:04,916 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:05,467 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:05,917 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:06,174 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:54:06,174 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:06,174 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:06,216 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 20:54:06,216 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 20:54:06,380 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval
+2025-01-06 20:54:06,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table
+2025-01-06 20:54:06,701 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/t9sie1ly-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-06 20:54:06,714 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/qnpv54y7-events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:54:06,729 INFO    wandb-upload_1:2115858 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppyoaom4f
+2025-01-06 20:54:06,791 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 20:54:06,800 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 20:54:06,909 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:07,141 INFO    SenderThread:2115858 [sender.py:send_request_log_artifact():1525] logged artifact run-bkletf5j-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM0MTgxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY1Mzc2Mg==', 'latestArtifact': None}}
+2025-01-06 20:54:07,141 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:54:07,141 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now
+2025-01-06 20:54:07,141 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:54:07,142 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,143 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,143 DEBUG   SenderThread:2115858 [sender.py:send():379] send: telemetry
+2025-01-06 20:54:07,143 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,143 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,143 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,144 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,144 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,144 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,144 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,146 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,146 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,146 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,146 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,147 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,147 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,147 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,147 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,148 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,148 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,148 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,148 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,148 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,149 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,149 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:07,149 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 20:54:07,376 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/0n8y3ctg-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-06 20:54:07,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:07,468 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:07,469 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:54:07,469 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json
+2025-01-06 20:54:07,469 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval
+2025-01-06 20:54:07,522 INFO    wandb-upload_1:2115858 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprhug1a3e
+2025-01-06 20:54:07,650 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/fkyioxav-media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json
+2025-01-06 20:54:07,909 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:07,963 INFO    SenderThread:2115858 [sender.py:send_request_log_artifact():1525] logged artifact run-bkletf5j-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM0MTg0OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY1Mzc2NQ==', 'latestArtifact': None}}
+2025-01-06 20:54:07,963 DEBUG   SenderThread:2115858 [sender.py:send():379] send: files
+2025-01-06 20:54:07,963 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:54:07,964 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json with policy now
+2025-01-06 20:54:07,964 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:54:07,964 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:07,964 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:08,210 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/dxsdmipb-media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json
+2025-01-06 20:54:08,469 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:08,909 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:09,469 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:09,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:10,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:11,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:12,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:13,835 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:54:13,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:14,910 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:15,725 DEBUG   SenderThread:2115858 [sender.py:send():379] send: exit
+2025-01-06 20:54:15,725 INFO    SenderThread:2115858 [sender.py:send_exit():586] handling exit code: 0
+2025-01-06 20:54:15,726 INFO    SenderThread:2115858 [sender.py:send_exit():588] handling runtime: 251
+2025-01-06 20:54:15,726 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:15,726 INFO    SenderThread:2115858 [sender.py:send_exit():594] send defer
+2025-01-06 20:54:15,726 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:15,726 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-06 20:54:15,726 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:15,726 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-06 20:54:15,726 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 1
+2025-01-06 20:54:15,726 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:15,727 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-06 20:54:15,727 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:15,727 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-06 20:54:15,727 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 2
+2025-01-06 20:54:15,727 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:15,727 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-06 20:54:15,727 INFO    HandlerThread:2115858 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-06 20:54:15,727 DEBUG   SystemMonitor:2115858 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-06 20:54:15,727 DEBUG   SystemMonitor:2115858 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-06 20:54:15,728 INFO    HandlerThread:2115858 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-06 20:54:15,728 INFO    HandlerThread:2115858 [interfaces.py:finish():200] Joined disk monitor
+2025-01-06 20:54:15,781 INFO    HandlerThread:2115858 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-06 20:54:15,781 INFO    HandlerThread:2115858 [interfaces.py:finish():200] Joined memory monitor
+2025-01-06 20:54:15,781 INFO    HandlerThread:2115858 [interfaces.py:finish():200] Joined network monitor
+2025-01-06 20:54:15,781 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:15,781 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-06 20:54:15,781 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 3
+2025-01-06 20:54:15,782 DEBUG   SenderThread:2115858 [sender.py:send():379] send: stats
+2025-01-06 20:54:15,782 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:15,782 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-06 20:54:15,782 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:15,782 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-06 20:54:15,782 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 4
+2025-01-06 20:54:15,782 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:15,782 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-06 20:54:16,472 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:17,472 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:22,179 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 20:54:22,179 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:54:22,179 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-06 20:54:22,180 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:22,180 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-06 20:54:22,180 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 5
+2025-01-06 20:54:22,180 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 20:54:22,180 DEBUG   SenderThread:2115858 [sender.py:send():379] send: history
+2025-01-06 20:54:22,180 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 20:54:22,181 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:22,181 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-06 20:54:22,181 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:22,182 DEBUG   SenderThread:2115858 [sender.py:send():379] send: summary
+2025-01-06 20:54:22,185 INFO    SenderThread:2115858 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 20:54:22,186 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:22,186 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-06 20:54:22,186 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 6
+2025-01-06 20:54:22,186 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:22,186 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-06 20:54:22,186 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:22,186 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-06 20:54:22,190 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 20:54:22,301 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 7
+2025-01-06 20:54:22,302 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:22,302 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-06 20:54:22,302 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:22,302 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-06 20:54:22,475 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:22,475 INFO    Thread-12 :2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml
+2025-01-06 20:54:22,727 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 20:54:24,861 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 8
+2025-01-06 20:54:24,861 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 20:54:24,861 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:24,861 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-06 20:54:24,862 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:24,862 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-06 20:54:24,862 INFO    SenderThread:2115858 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-06 20:54:24,864 INFO    SenderThread:2115858 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-06 20:54:24,866 INFO    SenderThread:2115858 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-06 20:54:24,881 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 9
+2025-01-06 20:54:24,881 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:24,881 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-06 20:54:24,881 DEBUG   SenderThread:2115858 [sender.py:send():379] send: artifact
+2025-01-06 20:54:25,113 INFO    SenderThread:2115858 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}}
+2025-01-06 20:54:25,113 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:25,113 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-06 20:54:25,113 INFO    SenderThread:2115858 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml config.yaml
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json wandb-summary.json
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0 events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/requirements.txt requirements.txt
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-metadata.json wandb-metadata.json
+2025-01-06 20:54:25,476 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log output.log
+2025-01-06 20:54:25,477 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/diff.patch diff.patch
+2025-01-06 20:54:25,480 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-06 20:54:25,482 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4 media/videos/eval/3_eval_3_8f993ed9f4bf0eae5cf1.mp4
+2025-01-06 20:54:25,483 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4 media/videos/eval/4_eval_4_6a8935b2358637140fbc.mp4
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4 media/videos/eval/1_eval_1_b0cb5720b5b83600f051.mp4
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4 media/videos/eval/0_eval_0_d5629c4a8c29fc9b2a6b.mp4
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4 media/videos/eval/2_eval_2_4cf0e0208d2fc5ae5937.mp4
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json media/table/eval/total_success_rate_ci_6_4142c490068220aa5597.table.json
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json
+2025-01-06 20:54:25,485 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 10
+2025-01-06 20:54:25,485 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:25,485 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-06 20:54:25,486 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:25,486 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-06 20:54:25,486 INFO    SenderThread:2115858 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 20:54:25,663 INFO    wandb-upload_1:2115858 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/wandb-summary.json
+2025-01-06 20:54:25,668 INFO    wandb-upload_2:2115858 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/config.yaml
+2025-01-06 20:54:25,705 INFO    wandb-upload_3:2115858 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/requirements.txt
+2025-01-06 20:54:25,719 INFO    wandb-upload_0:2115858 [upload_job.py:push():130] Uploaded file /tmp/tmp8_uk1vdtwandb/sacbiy0i-events.out.tfevents.1736214604.leonmkim-ROG-Strix-G15CS-G15CS.2115685.0
+2025-01-06 20:54:25,728 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 20:54:25,728 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 20:54:25,769 INFO    wandb-upload_4:2115858 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/files/output.log
+2025-01-06 20:54:25,969 INFO    Thread-11 (_thread_body):2115858 [sender.py:transition_state():614] send defer: 11
+2025-01-06 20:54:25,969 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:25,969 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-06 20:54:25,969 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:25,970 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-06 20:54:25,970 INFO    SenderThread:2115858 [file_pusher.py:join():175] waiting for file pusher
+2025-01-06 20:54:25,971 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 12
+2025-01-06 20:54:25,971 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:25,971 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-06 20:54:25,971 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:25,971 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-06 20:54:25,971 INFO    SenderThread:2115858 [file_stream.py:finish():601] file stream finish called
+2025-01-06 20:54:26,207 INFO    SenderThread:2115858 [file_stream.py:finish():605] file stream finish is done
+2025-01-06 20:54:26,207 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 13
+2025-01-06 20:54:26,207 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:26,207 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-06 20:54:26,207 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:26,207 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-06 20:54:26,207 INFO    SenderThread:2115858 [sender.py:transition_state():614] send defer: 14
+2025-01-06 20:54:26,207 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 20:54:26,207 INFO    HandlerThread:2115858 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-06 20:54:26,208 DEBUG   SenderThread:2115858 [sender.py:send():379] send: final
+2025-01-06 20:54:26,208 DEBUG   SenderThread:2115858 [sender.py:send():379] send: footer
+2025-01-06 20:54:26,208 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: defer
+2025-01-06 20:54:26,208 INFO    SenderThread:2115858 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-06 20:54:26,208 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 20:54:26,208 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 20:54:26,209 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: server_info
+2025-01-06 20:54:26,209 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: server_info
+2025-01-06 20:54:26,210 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 20:54:26,210 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-06 20:54:26,211 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-06 20:54:26,211 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 20:54:26,262 DEBUG   SenderThread:2115858 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 20:54:26,262 INFO    MainThread:2115858 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-06 20:54:26,263 INFO    MainThread:2115858 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-06 20:54:26,263 INFO    MainThread:2115858 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-06 20:54:26,263 DEBUG   HandlerThread:2115858 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-06 20:54:26,263 INFO    HandlerThread:2115858 [handler.py:finish():882] shutting down handler
+2025-01-06 20:54:27,210 INFO    WriterThread:2115858 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/run-bkletf5j.wandb
+2025-01-06 20:54:27,262 INFO    SenderThread:2115858 [sender.py:finish():1615] shutting down sender
+2025-01-06 20:54:27,262 INFO    SenderThread:2115858 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 20:54:27,262 INFO    SenderThread:2115858 [file_pusher.py:join():175] waiting for file pusher
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/logs/debug.log b/bkletf5j/wandb/run-20250106_205004-bkletf5j/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..09ef718cf064dc90d0ff3e0f9084528f7c2137a2
--- /dev/null
+++ b/bkletf5j/wandb/run-20250106_205004-bkletf5j/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Configure stats pid to 2115685
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/settings
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-06 20:50:04,128 INFO    MainThread:2115685 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/logs/debug.log
+2025-01-06 20:50:04,129 INFO    MainThread:2115685 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/wandb/run-20250106_205004-bkletf5j/logs/debug-internal.log
+2025-01-06 20:50:04,129 INFO    MainThread:2115685 [wandb_init.py:init():569] calling init triggers
+2025-01-06 20:50:04,129 INFO    MainThread:2115685 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j'}
+2025-01-06 20:50:04,129 INFO    MainThread:2115685 [wandb_init.py:init():619] starting backend
+2025-01-06 20:50:04,129 INFO    MainThread:2115685 [wandb_init.py:init():623] setting up manager
+2025-01-06 20:50:04,133 INFO    MainThread:2115685 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-06 20:50:04,134 INFO    MainThread:2115685 [wandb_init.py:init():631] backend started and connected
+2025-01-06 20:50:04,146 INFO    MainThread:2115685 [wandb_init.py:init():720] updated telemetry
+2025-01-06 20:50:04,153 INFO    MainThread:2115685 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-06 20:50:04,401 INFO    MainThread:2115685 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-06 20:50:04,470 INFO    MainThread:2115685 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-06 20:50:04,470 INFO    MainThread:2115685 [wandb_init.py:init():804] starting run threads in backend
+2025-01-06 20:50:04,823 INFO    MainThread:2115685 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-06 20:50:04,823 INFO    MainThread:2115685 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-06 20:50:04,823 INFO    MainThread:2115685 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-06 20:50:04,823 INFO    MainThread:2115685 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-06 20:50:04,825 INFO    MainThread:2115685 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-06 20:50:04,825 INFO    MainThread:2115685 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bkletf5j/tb, True
+2025-01-06 20:50:09,543 INFO    MainThread:2115685 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'}
+2025-01-06 20:54:27,264 WARNING MsgRouterThr:2115685 [router.py:message_loop():77] message_loop has been closed
diff --git a/bkletf5j/wandb/run-20250106_205004-bkletf5j/run-bkletf5j.wandb b/bkletf5j/wandb/run-20250106_205004-bkletf5j/run-bkletf5j.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..66189ea5c964cffb04c6fca486a5e46907cfef44
Binary files /dev/null and b/bkletf5j/wandb/run-20250106_205004-bkletf5j/run-bkletf5j.wandb differ
diff --git a/c9fefav0/episode_rosbags/episode_0_2025-01-06-21-06-27.bag b/c9fefav0/episode_rosbags/episode_0_2025-01-06-21-06-27.bag
new file mode 100644
index 0000000000000000000000000000000000000000..5aee5f154287ab0da6444452f2555a5fd645345d
--- /dev/null
+++ b/c9fefav0/episode_rosbags/episode_0_2025-01-06-21-06-27.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4a90ede835421d3ec0469be5ffb395c76fecf34e60fd245062f1c8d8d981874
+size 1412264562
diff --git a/c9fefav0/episode_rosbags/episode_4_2025-01-06-21-09-26.bag b/c9fefav0/episode_rosbags/episode_4_2025-01-06-21-09-26.bag
new file mode 100644
index 0000000000000000000000000000000000000000..72c536ca617c81c5295ed4a7c72667cc78f5a5d3
--- /dev/null
+++ b/c9fefav0/episode_rosbags/episode_4_2025-01-06-21-09-26.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17acce24266ff7501a748bd203eb5b657811cee46ab740b471102ad56bb760e4
+size 1413885504
diff --git a/kvtik1hq/.hydra/config.yaml b/kvtik1hq/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804
--- /dev/null
+++ b/kvtik1hq/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1002_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/kvtik1hq/.hydra/hydra.yaml b/kvtik1hq/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1a054f6de1bf5888e9e8b99e938099d0d637fc5e
--- /dev/null
+++ b/kvtik1hq/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kvtik1hq
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/kvtik1hq/.hydra/overrides.yaml b/kvtik1hq/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/kvtik1hq/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/kvtik1hq/eval_policy.log b/kvtik1hq/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..d1bfc9a62f478a0afeb45d0e6fd40ca04f601ec2
--- /dev/null
+++ b/kvtik1hq/eval_policy.log
@@ -0,0 +1,15 @@
+[2025-01-07 17:18:23,136][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-07 17:18:23,139][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-07 17:18:26,989][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+
diff --git a/pmcil3p3/.hydra/config.yaml b/pmcil3p3/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804
--- /dev/null
+++ b/pmcil3p3/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1002_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/pmcil3p3/.hydra/hydra.yaml b/pmcil3p3/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2f78115dc8f92205139554ce22d8952c953085a5
--- /dev/null
+++ b/pmcil3p3/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/pmcil3p3
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/pmcil3p3/.hydra/overrides.yaml b/pmcil3p3/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/pmcil3p3/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/pmcil3p3/eval_policy.log b/pmcil3p3/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..90d887f1c10f1ef031768d20e950a2382ca01782
--- /dev/null
+++ b/pmcil3p3/eval_policy.log
@@ -0,0 +1,15 @@
+[2025-01-06 20:49:50,488][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-06 20:49:50,492][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-06 20:49:51,671][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+