diff --git a/.gitattributes b/.gitattributes
index 1496f5b7b49b8e4d2257950f842a62e78c24ba05..57800db0c512024f59a8f294a9e3383661d39cb7 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -68,3 +68,54 @@ w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/1_eval_1_f49
 w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 filter=lfs diff=lfs merge=lfs -text
 w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 filter=lfs diff=lfs merge=lfs -text
 w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 filter=lfs diff=lfs merge=lfs -text
+w3dz1sd1/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+w3dz1sd1/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+w3dz1sd1/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+w3dz1sd1/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+5qmtbx9d/episode_rosbags/episode_4_2025-01-07-20-48-21.bag filter=lfs diff=lfs merge=lfs -text
+w3dz1sd1/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nyyc47dd/episode_rosbags/episode_3_2025-01-07-17-12-13.bag filter=lfs diff=lfs merge=lfs -text
+nyyc47dd/episode_rosbags/episode_2_2025-01-07-17-11-29.bag filter=lfs diff=lfs merge=lfs -text
+9jw2ghih/episode_rosbags/episode_2_2025-01-06-20-42-37.bag filter=lfs diff=lfs merge=lfs -text
+9jw2ghih/episode_rosbags/episode_1_2025-01-06-20-41-55.bag filter=lfs diff=lfs merge=lfs -text
+w3dz1sd1/episode_rosbags/episode_1_2025-01-07-17-47-21.bag filter=lfs diff=lfs merge=lfs -text
+9jw2ghih/episode_rosbags/episode_3_2025-01-06-20-43-18.bag filter=lfs diff=lfs merge=lfs -text
+nyyc47dd/episode_rosbags/episode_4_2025-01-07-17-12-54.bag filter=lfs diff=lfs merge=lfs -text
+5qmtbx9d/episode_rosbags/episode_3_2025-01-07-20-47-34.bag filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4 filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4 filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4 filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4 filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4 filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+5qmtbx9d/episode_rosbags/episode_0_2025-01-07-20-45-18.bag filter=lfs diff=lfs merge=lfs -text
+5qmtbx9d/episode_rosbags/episode_2_2025-01-07-20-46-48.bag filter=lfs diff=lfs merge=lfs -text
+w3dz1sd1/episode_rosbags/episode_2_2025-01-07-17-48-01.bag filter=lfs diff=lfs merge=lfs -text
+9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4 filter=lfs diff=lfs merge=lfs -text
+9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4 filter=lfs diff=lfs merge=lfs -text
+w3dz1sd1/episode_rosbags/episode_0_2025-01-07-17-46-27.bag filter=lfs diff=lfs merge=lfs -text
+9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4 filter=lfs diff=lfs merge=lfs -text
+9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4 filter=lfs diff=lfs merge=lfs -text
+9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4 filter=lfs diff=lfs merge=lfs -text
+9ig9megf/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9ig9megf/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9ig9megf/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9ig9megf/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9ig9megf/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nyyc47dd/episode_rosbags/episode_0_2025-01-07-17-10-04.bag filter=lfs diff=lfs merge=lfs -text
+6qq3p71c/episode_rosbags/episode_4_2025-01-07-17-17-13.bag filter=lfs diff=lfs merge=lfs -text
+9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4 filter=lfs diff=lfs merge=lfs -text
+9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4 filter=lfs diff=lfs merge=lfs -text
+9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4 filter=lfs diff=lfs merge=lfs -text
+9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4 filter=lfs diff=lfs merge=lfs -text
+9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4 filter=lfs diff=lfs merge=lfs -text
+9fviy2t8/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9fviy2t8/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9fviy2t8/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9fviy2t8/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9fviy2t8/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+9jw2ghih/episode_rosbags/episode_0_2025-01-06-20-41-14.bag filter=lfs diff=lfs merge=lfs -text
diff --git a/2inskug9/.hydra/config.yaml b/2inskug9/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab
--- /dev/null
+++ b/2inskug9/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1009_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/2inskug9/.hydra/hydra.yaml b/2inskug9/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1062e200813abe043fdd5ebdf4cbdbeee15fbe30
--- /dev/null
+++ b/2inskug9/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/2inskug9
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/2inskug9/.hydra/overrides.yaml b/2inskug9/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/2inskug9/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/2inskug9/eval_policy.log b/2inskug9/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..d574aa8325b8685a467045b51038633b4fb3857b
--- /dev/null
+++ b/2inskug9/eval_policy.log
@@ -0,0 +1,15 @@
+[2025-01-06 20:40:45,800][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-06 20:40:45,803][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-06 20:40:49,726][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+
diff --git a/5qmtbx9d/episode_rosbags/episode_0_2025-01-07-20-45-18.bag b/5qmtbx9d/episode_rosbags/episode_0_2025-01-07-20-45-18.bag
new file mode 100644
index 0000000000000000000000000000000000000000..794a17b90d09c86095c9ef16ef18045cf4286464
--- /dev/null
+++ b/5qmtbx9d/episode_rosbags/episode_0_2025-01-07-20-45-18.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe23231707dc8cd039df6798393bb32a7fdb2b5269f0d7491643a090c40e820c
+size 1406423449
diff --git a/5qmtbx9d/episode_rosbags/episode_2_2025-01-07-20-46-48.bag b/5qmtbx9d/episode_rosbags/episode_2_2025-01-07-20-46-48.bag
new file mode 100644
index 0000000000000000000000000000000000000000..1f29da801c79c7835af6ad8681731d7e2a528e14
--- /dev/null
+++ b/5qmtbx9d/episode_rosbags/episode_2_2025-01-07-20-46-48.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2059801822bb12b3c7ec5ce54a24944d9f97192cc5e05bc9e28bef29b24b0fa5
+size 1412262733
diff --git a/5qmtbx9d/episode_rosbags/episode_3_2025-01-07-20-47-34.bag b/5qmtbx9d/episode_rosbags/episode_3_2025-01-07-20-47-34.bag
new file mode 100644
index 0000000000000000000000000000000000000000..0ca24f17420b242d6209f1b229148c69603f8c31
--- /dev/null
+++ b/5qmtbx9d/episode_rosbags/episode_3_2025-01-07-20-47-34.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ed878bad62a79fc9dea7f7e774e1abd90b0816780c39be281c5de5c644ed698
+size 1410566127
diff --git a/5qmtbx9d/episode_rosbags/episode_4_2025-01-07-20-48-21.bag b/5qmtbx9d/episode_rosbags/episode_4_2025-01-07-20-48-21.bag
new file mode 100644
index 0000000000000000000000000000000000000000..066ee4edcc69ea62925decc1aa926c9099e11f39
--- /dev/null
+++ b/5qmtbx9d/episode_rosbags/episode_4_2025-01-07-20-48-21.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07195c6d41a9d448314956089995708a527c9c4e8e962f85d9c31900aa54af47
+size 1410737371
diff --git a/6qq3p71c/.hydra/config.yaml b/6qq3p71c/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab
--- /dev/null
+++ b/6qq3p71c/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1009_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/6qq3p71c/.hydra/hydra.yaml b/6qq3p71c/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..f9fb22c122b5b9368793cda18c7c1ba40cbebeec
--- /dev/null
+++ b/6qq3p71c/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/6qq3p71c/.hydra/overrides.yaml b/6qq3p71c/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/6qq3p71c/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/6qq3p71c/episode_rosbags/aligned_depth_to_color_K.npy b/6qq3p71c/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/6qq3p71c/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/6qq3p71c/episode_rosbags/cam_tf_world.npy b/6qq3p71c/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/6qq3p71c/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/6qq3p71c/episode_rosbags/color_K.npy b/6qq3p71c/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/6qq3p71c/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/6qq3p71c/episode_rosbags/depth_K.npy b/6qq3p71c/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/6qq3p71c/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/6qq3p71c/episode_rosbags/episode_4_2025-01-07-17-17-13.bag b/6qq3p71c/episode_rosbags/episode_4_2025-01-07-17-17-13.bag
new file mode 100644
index 0000000000000000000000000000000000000000..3ab77e62372a8602d93cb0d425133c3eb86aa305
--- /dev/null
+++ b/6qq3p71c/episode_rosbags/episode_4_2025-01-07-17-17-13.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2158b2221e341a2b988d7b18ed824b65166d5c9e6a8b4eff757e59ca1e1254f3
+size 1411049355
diff --git a/6qq3p71c/eval_robot.log b/6qq3p71c/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..1ba96aa921b345af4bd6806816f459f7c2d3ddb9
--- /dev/null
+++ b/6qq3p71c/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-07 17:13:58,662][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-07 17:13:58,665][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/6qq3p71c/eval_video/0_eval.mp4 b/6qq3p71c/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9a533ef26f078d4ff471af6ec3b50356e77fdbf3
--- /dev/null
+++ b/6qq3p71c/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:393b812fc042bd7e1762bb31f89641f23d83d9a7bccf663152ad6f2502cdd75c
+size 1128747
diff --git a/6qq3p71c/eval_video/1_eval.mp4 b/6qq3p71c/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..7a9d1a2b9053e50da96cd447e157abf8d2469657
--- /dev/null
+++ b/6qq3p71c/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19268ebe421ff04fe7f07cba82a91497bcf023a2d0526110b2982654792f301b
+size 1142439
diff --git a/6qq3p71c/eval_video/2_eval.mp4 b/6qq3p71c/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..da947de61c8e19ef8a1466dd2caf38799d64af9c
--- /dev/null
+++ b/6qq3p71c/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d676baf746a2d4745effc19bc6a12a19adb6dfa8742a5b6b0537f33e50c77f52
+size 1148399
diff --git a/6qq3p71c/eval_video/3_eval.mp4 b/6qq3p71c/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..3ce05e3ce0935a36ccb73738d0c10e9d62fa6829
--- /dev/null
+++ b/6qq3p71c/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c26e92b3a127f92ec68d16c302727ae990607e8ee5c7ecfaca76e7ea315c3ec
+size 1146191
diff --git a/6qq3p71c/eval_video/4_eval.mp4 b/6qq3p71c/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5ce06b5b50a7c37f2db9e060253486930008e777
--- /dev/null
+++ b/6qq3p71c/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d693180a6d9e048e7c1db45a7f0a3fa444e0f4e9a22a930e02f109758873c7f4
+size 1139534
diff --git a/6qq3p71c/tb/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0 b/6qq3p71c/tb/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
new file mode 100644
index 0000000000000000000000000000000000000000..2ea2f9015a187a2424c9b201f9109961f54e51fd
--- /dev/null
+++ b/6qq3p71c/tb/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc7e0708cc13bc2c501e8c26d0c19cbc66025f21d3961465790db06caa2622c0
+size 6539
diff --git a/6qq3p71c/wandb/debug-internal.log b/6qq3p71c/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..64fb7bfe5869f22ea7cba83b21d9af77d392dc28
--- /dev/null
+++ b/6qq3p71c/wandb/debug-internal.log
@@ -0,0 +1,716 @@
+2025-01-07 17:14:04,431 INFO    StreamThr :2196222 [internal.py:wandb_internal():85] W&B internal server running at pid: 2196222, started at: 2025-01-07 17:14:04.431363
+2025-01-07 17:14:04,432 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status
+2025-01-07 17:14:04,443 INFO    WriterThread:2196222 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/run-6qq3p71c.wandb
+2025-01-07 17:14:04,444 DEBUG   SenderThread:2196222 [sender.py:send():379] send: header
+2025-01-07 17:14:04,455 DEBUG   SenderThread:2196222 [sender.py:send():379] send: run
+2025-01-07 17:14:04,686 INFO    SenderThread:2196222 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files
+2025-01-07 17:14:04,687 INFO    SenderThread:2196222 [sender.py:_start_run_threads():1188] run started: 6qq3p71c with start time 1736288044.431686
+2025-01-07 17:14:04,702 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: check_version
+2025-01-07 17:14:04,702 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: check_version
+2025-01-07 17:14:04,776 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: run_start
+2025-01-07 17:14:04,780 DEBUG   HandlerThread:2196222 [system_info.py:__init__():26] System info init
+2025-01-07 17:14:04,780 DEBUG   HandlerThread:2196222 [system_info.py:__init__():41] System info init done
+2025-01-07 17:14:04,780 INFO    HandlerThread:2196222 [system_monitor.py:start():194] Starting system monitor
+2025-01-07 17:14:04,781 INFO    SystemMonitor:2196222 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-07 17:14:04,781 INFO    HandlerThread:2196222 [system_monitor.py:probe():214] Collecting system info
+2025-01-07 17:14:04,781 INFO    SystemMonitor:2196222 [interfaces.py:start():188] Started cpu monitoring
+2025-01-07 17:14:04,782 INFO    SystemMonitor:2196222 [interfaces.py:start():188] Started disk monitoring
+2025-01-07 17:14:04,783 INFO    SystemMonitor:2196222 [interfaces.py:start():188] Started gpu monitoring
+2025-01-07 17:14:04,783 INFO    SystemMonitor:2196222 [interfaces.py:start():188] Started memory monitoring
+2025-01-07 17:14:04,784 INFO    SystemMonitor:2196222 [interfaces.py:start():188] Started network monitoring
+2025-01-07 17:14:04,794 DEBUG   HandlerThread:2196222 [system_info.py:probe():152] Probing system
+2025-01-07 17:14:04,796 DEBUG   HandlerThread:2196222 [system_info.py:_probe_git():137] Probing git
+2025-01-07 17:14:04,800 DEBUG   HandlerThread:2196222 [system_info.py:_probe_git():145] Probing git done
+2025-01-07 17:14:04,800 DEBUG   HandlerThread:2196222 [system_info.py:probe():200] Probing system done
+2025-01-07 17:14:04,800 DEBUG   HandlerThread:2196222 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:14:04.794650', 'startedAt': '2025-01-07T22:14:04.419165', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.5576249999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.613, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.631, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 743.5787963867188}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-07 17:14:04,800 INFO    HandlerThread:2196222 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-07 17:14:04,801 INFO    HandlerThread:2196222 [system_monitor.py:probe():227] Publishing system info
+2025-01-07 17:14:04,801 DEBUG   HandlerThread:2196222 [system_info.py:_save_code():44] Saving code
+2025-01-07 17:14:04,806 DEBUG   HandlerThread:2196222 [system_info.py:_save_code():67] Saving code done
+2025-01-07 17:14:04,806 DEBUG   HandlerThread:2196222 [system_info.py:_save_patches():84] Saving git patches
+2025-01-07 17:14:04,876 DEBUG   HandlerThread:2196222 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-07 17:14:04,877 INFO    HandlerThread:2196222 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-07 17:14:04,880 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:14:04,880 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-07 17:14:04,880 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-07 17:14:04,880 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-07 17:14:05,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-07 17:14:05,123 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: python_packages
+2025-01-07 17:14:05,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:14:05,123 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:14:05,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:05,126 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/pm4k7w63-wandb-metadata.json
+2025-01-07 17:14:05,126 INFO    HandlerThread:2196222 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/tb"
+  save: true
+}
+control {
+  relay_id: "38255728e4c045ed8f1dd189a41437c0"
+}
+_info {
+  stream_id: "6qq3p71c"
+}
+
+2025-01-07 17:14:05,126 DEBUG   HandlerThread:2196222 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-07 17:14:05,178 INFO    wandb-upload_1:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/nl9lvu9s-code/FISH/eval_robot.py
+2025-01-07 17:14:05,187 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/kbip47ap-diff.patch
+2025-01-07 17:14:05,239 DEBUG   SenderThread:2196222 [sender.py:send():379] send: telemetry
+2025-01-07 17:14:05,239 DEBUG   SenderThread:2196222 [sender.py:send():379] send: tbrecord
+2025-01-07 17:14:05,240 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:14:05,240 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0 with policy live
+2025-01-07 17:14:05,432 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/crfyvop6-events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:14:05,688 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code/FISH/eval_robot.py
+2025-01-07 17:14:05,688 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-metadata.json
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/requirements.txt
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/diff.patch
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code/FISH
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code
+2025-01-07 17:14:06,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:07,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:07,445 DEBUG   SenderThread:2196222 [sender.py:send():379] send: config
+2025-01-07 17:14:07,688 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:08,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:09,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:09,497 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:09,688 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:10,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:11,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:12,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:13,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:13,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:14,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:14,885 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:15,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:16,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:17,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:17,690 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:18,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:19,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:20,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:14:20,123 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:14:20,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:20,250 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:21,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:21,692 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:22,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:23,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:23,692 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:24,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:25,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:25,693 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:26,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:26,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:27,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:28,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:29,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:30,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:31,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:31,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:32,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:33,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:34,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:35,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:14:35,123 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:14:35,165 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:36,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:36,225 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:36,696 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml
+2025-01-07 17:14:37,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:38,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:39,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:40,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:41,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:42,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:42,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:43,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:44,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:45,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:46,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:47,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:48,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:48,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:49,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:50,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:14:50,125 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:14:50,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:50,423 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:14:50,423 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4 with policy now
+2025-01-07 17:14:50,423 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:14:50,424 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:14:50,424 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:14:50,426 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:14:50,673 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/wmm0r09y-media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4
+2025-01-07 17:14:50,699 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:14:50,700 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4
+2025-01-07 17:14:50,700 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media
+2025-01-07 17:14:50,700 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval
+2025-01-07 17:14:50,701 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos
+2025-01-07 17:14:50,968 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/n6l1p8xt-media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4
+2025-01-07 17:14:51,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:51,700 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:52,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:53,130 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:53,430 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:54,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:55,130 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:55,701 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:56,130 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:57,130 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:57,702 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:58,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:58,681 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:59,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:00,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:01,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:01,703 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:15:02,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:03,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:03,704 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:04,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:04,593 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:04,784 DEBUG   SystemMonitor:2196222 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-07 17:15:04,785 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:15:05,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:15:05,127 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:15:05,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:06,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:07,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:07,705 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:08,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:09,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:09,861 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:10,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:11,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:11,706 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:12,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:13,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:14,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:15,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:15,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:16,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:17,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:18,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:19,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:20,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:15:20,125 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:15:20,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:20,239 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:21,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:22,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:23,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:24,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:25,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:25,711 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:26,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:26,134 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:27,134 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:28,134 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:29,134 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:30,135 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:31,136 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:32,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:32,136 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:33,136 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:33,445 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:15:33,445 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:15:33,445 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4 with policy now
+2025-01-07 17:15:33,446 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:15:33,446 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:15:33,447 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:15:33,706 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/zg8yj1wa-media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4
+2025-01-07 17:15:33,713 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:33,714 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:15:33,714 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4
+2025-01-07 17:15:33,714 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval
+2025-01-07 17:15:33,980 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/3n0ami7s-media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4
+2025-01-07 17:15:34,138 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:34,785 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:15:35,130 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:15:35,130 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:15:35,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:35,714 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:36,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:37,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:37,993 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:38,144 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:39,143 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:39,715 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:40,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:41,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:41,716 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:42,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:43,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:43,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:44,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:44,518 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:15:44,518 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:15:44,519 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:15:44,717 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:15:44,717 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:15:44,978 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/pzupy8xk-events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:15:45,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:45,718 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:46,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:47,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:47,718 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:48,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:48,953 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:49,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:49,719 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:50,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:15:50,126 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:15:50,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:51,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:52,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:53,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:54,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:54,293 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:55,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:56,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:57,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:58,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:59,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:00,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:00,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:01,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:02,144 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:03,143 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:04,144 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:04,786 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:16:05,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:16:05,126 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:16:05,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:05,222 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:05,725 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:06,144 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:07,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:08,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:09,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:10,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:11,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:11,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:12,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:13,146 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:14,146 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:15,149 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:15,728 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:15,950 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:16:15,950 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4 with policy now
+2025-01-07 17:16:15,958 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:16:15,958 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:16:15,958 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:16:15,959 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:16:16,150 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:16,202 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/4er4aazk-media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4
+2025-01-07 17:16:16,728 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:16:16,728 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4
+2025-01-07 17:16:16,728 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval
+2025-01-07 17:16:16,982 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:17,009 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/3i7cr54k-media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4
+2025-01-07 17:16:17,151 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:17,729 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:18,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:18,729 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:19,150 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:20,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:16:20,129 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:16:20,152 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:21,154 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:21,731 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:22,152 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:22,272 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:23,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:23,731 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:24,156 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:25,155 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:26,157 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:27,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:28,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:28,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:29,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:29,733 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:16:30,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:30,354 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:16:30,354 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:16:30,355 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:16:30,734 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:16:31,158 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:31,734 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:32,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:33,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:33,396 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:34,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:34,786 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:16:35,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:16:35,126 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:16:35,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:35,735 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:36,158 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:37,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:38,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:39,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:39,215 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:40,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:41,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:42,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:43,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:44,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:45,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:45,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:46,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:47,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:48,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:49,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:50,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:16:50,126 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:16:50,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:50,291 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:51,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:52,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:53,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:54,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:55,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:56,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:56,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:57,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:58,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:59,164 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:00,164 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:01,165 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:01,317 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:01,661 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:17:01,661 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4 with policy now
+2025-01-07 17:17:01,662 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:17:01,663 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:01,663 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:01,663 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:01,744 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:01,744 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:01,744 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4
+2025-01-07 17:17:01,745 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval
+2025-01-07 17:17:01,922 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/qf3aacrc-media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4
+2025-01-07 17:17:02,164 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:02,189 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/pjenyrtw-media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4
+2025-01-07 17:17:03,165 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:03,745 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:04,167 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:04,787 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:17:05,138 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:17:05,138 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:17:05,185 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:06,167 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:07,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:07,259 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:07,746 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:08,167 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:09,168 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:09,747 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:09,747 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:17:10,000 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/d6zk95u6-events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:17:10,168 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:10,406 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:10,406 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:10,406 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:10,747 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:11,171 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:11,748 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:12,171 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:12,585 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:13,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:14,168 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:15,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:15,749 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:16,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:17,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:18,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:18,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:19,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:20,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:17:20,128 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:17:20,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:21,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:22,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:23,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:23,224 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:24,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:25,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:26,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:27,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:28,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:29,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:29,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:30,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:31,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:32,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:33,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:34,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:34,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:34,787 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:17:35,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:17:35,128 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:17:35,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:36,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:37,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:38,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:39,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:39,222 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:40,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:41,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:41,758 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:41,822 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:17:41,822 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4 with policy now
+2025-01-07 17:17:41,827 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:17:41,828 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:41,828 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:41,828 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:42,078 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/32uoeb2q-media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4
+2025-01-07 17:17:42,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:42,758 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:42,758 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4
+2025-01-07 17:17:42,758 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval
+2025-01-07 17:17:43,070 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/s3n9j6oy-media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4
+2025-01-07 17:17:43,175 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:43,758 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:44,181 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:44,759 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:44,852 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:45,180 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:46,178 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:47,182 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:47,760 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:48,186 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:49,186 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:49,761 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:50,144 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:50,146 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:17:50,146 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:17:50,190 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:51,185 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:51,762 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:17:52,081 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 17:17:52,082 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 17:17:52,183 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:52,262 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:17:52,601 INFO    wandb-upload_2:2196222 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_hat441t
+2025-01-07 17:17:52,635 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 17:17:52,643 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:17:52,762 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:17:52,762 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:17:52,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json
+2025-01-07 17:17:52,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media
+2025-01-07 17:17:52,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval
+2025-01-07 17:17:52,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table
+2025-01-07 17:17:53,001 INFO    SenderThread:2196222 [sender.py:send_request_log_artifact():1525] logged artifact run-6qq3p71c-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkwOTQ1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY5NjEwOQ==', 'latestArtifact': None}}
+2025-01-07 17:17:53,001 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:17:53,001 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now
+2025-01-07 17:17:53,001 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:53,001 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,001 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,002 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:53,002 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,002 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,003 DEBUG   SenderThread:2196222 [sender.py:send():379] send: telemetry
+2025-01-07 17:17:53,003 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,003 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,003 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,003 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,004 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,004 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,004 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,004 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,005 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,005 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,005 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,005 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,005 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,006 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,006 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,006 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,006 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,007 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,007 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,007 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,007 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,007 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,008 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 17:17:53,177 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/6c7xiqr1-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:17:53,181 INFO    wandb-upload_1:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/wq18ni64-media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json
+2025-01-07 17:17:53,183 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:53,417 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/p7744o24-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:17:53,482 INFO    wandb-upload_2:2196222 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppgxspz31
+2025-01-07 17:17:53,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:53,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:53,842 INFO    SenderThread:2196222 [sender.py:send_request_log_artifact():1525] logged artifact run-6qq3p71c-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkwOTQ5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY5NjExMQ==', 'latestArtifact': None}}
+2025-01-07 17:17:53,842 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:17:53,842 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json with policy now
+2025-01-07 17:17:53,843 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:53,843 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,843 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:54,078 INFO    wandb-upload_1:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/hyz89z7w-media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json
+2025-01-07 17:17:54,184 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:54,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:55,184 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:56,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:56,185 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:56,764 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml
+2025-01-07 17:17:57,185 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:58,185 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:58,686 DEBUG   SenderThread:2196222 [sender.py:send():379] send: exit
+2025-01-07 17:17:58,686 INFO    SenderThread:2196222 [sender.py:send_exit():586] handling exit code: 0
+2025-01-07 17:17:58,686 INFO    SenderThread:2196222 [sender.py:send_exit():588] handling runtime: 233
+2025-01-07 17:17:58,686 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:58,686 INFO    SenderThread:2196222 [sender.py:send_exit():594] send defer
+2025-01-07 17:17:58,686 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:17:58,686 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-07 17:17:58,687 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:17:58,687 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-07 17:17:58,687 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 1
+2025-01-07 17:17:58,687 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:17:58,687 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-07 17:17:58,687 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:17:58,687 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-07 17:17:58,687 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 2
+2025-01-07 17:17:58,687 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:17:58,687 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-07 17:17:58,688 INFO    HandlerThread:2196222 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-07 17:17:58,688 DEBUG   SystemMonitor:2196222 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-07 17:17:58,688 DEBUG   SystemMonitor:2196222 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-07 17:17:58,688 INFO    HandlerThread:2196222 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-07 17:17:58,689 INFO    HandlerThread:2196222 [interfaces.py:finish():200] Joined disk monitor
+2025-01-07 17:17:58,740 INFO    HandlerThread:2196222 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-07 17:17:58,741 INFO    HandlerThread:2196222 [interfaces.py:finish():200] Joined memory monitor
+2025-01-07 17:17:58,741 INFO    HandlerThread:2196222 [interfaces.py:finish():200] Joined network monitor
+2025-01-07 17:17:58,741 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:17:58,741 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-07 17:17:58,741 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 3
+2025-01-07 17:17:58,741 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:17:58,742 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:17:58,742 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-07 17:17:58,742 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:17:58,742 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-07 17:17:58,742 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 4
+2025-01-07 17:17:58,742 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:17:58,742 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-07 17:17:58,765 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:59,765 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:18:05,460 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:18:05,461 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:18:05,461 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-07 17:18:05,462 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:05,462 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-07 17:18:05,462 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 5
+2025-01-07 17:18:05,462 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:05,462 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:18:05,462 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-07 17:18:05,462 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:18:05,463 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:18:05,464 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:18:05,464 DEBUG   SenderThread:2196222 [sender.py:send():379] send: summary
+2025-01-07 17:18:05,468 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:18:05,468 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:05,468 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-07 17:18:05,468 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 6
+2025-01-07 17:18:05,468 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:05,468 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-07 17:18:05,469 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:05,469 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-07 17:18:05,469 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 7
+2025-01-07 17:18:05,469 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:18:05,469 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:05,469 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-07 17:18:05,469 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:05,469 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-07 17:18:05,687 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:18:05,768 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:18:05,928 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 8
+2025-01-07 17:18:05,928 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:18:05,928 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:05,928 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-07 17:18:05,928 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:05,928 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-07 17:18:05,928 INFO    SenderThread:2196222 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-07 17:18:05,931 INFO    SenderThread:2196222 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-07 17:18:05,933 INFO    SenderThread:2196222 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-07 17:18:05,946 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 9
+2025-01-07 17:18:05,946 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:05,946 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-07 17:18:05,946 DEBUG   SenderThread:2196222 [sender.py:send():379] send: artifact
+2025-01-07 17:18:06,151 INFO    SenderThread:2196222 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'versionIndex': 435}}}
+2025-01-07 17:18:06,151 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:06,151 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-07 17:18:06,151 INFO    SenderThread:2196222 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-07 17:18:06,687 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:18:06,768 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml config.yaml
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json wandb-summary.json
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/requirements.txt requirements.txt
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-metadata.json wandb-metadata.json
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0 events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log output.log
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/diff.patch diff.patch
+2025-01-07 17:18:06,774 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-07 17:18:06,776 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4 media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4
+2025-01-07 17:18:06,776 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4 media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4
+2025-01-07 17:18:06,776 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4 media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4
+2025-01-07 17:18:06,776 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4 media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4
+2025-01-07 17:18:06,777 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4 media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4
+2025-01-07 17:18:06,778 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json
+2025-01-07 17:18:06,778 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:18:06,778 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 10
+2025-01-07 17:18:06,778 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:18:06,778 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:06,779 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-07 17:18:06,779 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:06,779 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-07 17:18:06,779 INFO    SenderThread:2196222 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 17:18:06,951 INFO    wandb-upload_1:2196222 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/requirements.txt
+2025-01-07 17:18:06,952 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml
+2025-01-07 17:18:06,984 INFO    wandb-upload_4:2196222 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:18:07,000 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:18:07,045 INFO    wandb-upload_3:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/urz358fq-events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:18:07,246 INFO    Thread-11 (_thread_body):2196222 [sender.py:transition_state():614] send defer: 11
+2025-01-07 17:18:07,246 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:07,246 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-07 17:18:07,247 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:07,247 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-07 17:18:07,247 INFO    SenderThread:2196222 [file_pusher.py:join():175] waiting for file pusher
+2025-01-07 17:18:07,248 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 12
+2025-01-07 17:18:07,248 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:07,248 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-07 17:18:07,248 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:07,248 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-07 17:18:07,248 INFO    SenderThread:2196222 [file_stream.py:finish():601] file stream finish called
+2025-01-07 17:18:07,538 INFO    SenderThread:2196222 [file_stream.py:finish():605] file stream finish is done
+2025-01-07 17:18:07,538 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 13
+2025-01-07 17:18:07,538 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:07,538 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-07 17:18:07,538 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:07,538 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-07 17:18:07,539 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 14
+2025-01-07 17:18:07,539 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:07,539 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-07 17:18:07,539 DEBUG   SenderThread:2196222 [sender.py:send():379] send: final
+2025-01-07 17:18:07,539 DEBUG   SenderThread:2196222 [sender.py:send():379] send: footer
+2025-01-07 17:18:07,539 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:07,539 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-07 17:18:07,539 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:18:07,540 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:18:07,540 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: server_info
+2025-01-07 17:18:07,540 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: server_info
+2025-01-07 17:18:07,543 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:18:07,543 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-07 17:18:07,544 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-07 17:18:07,545 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:18:07,588 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:18:07,589 INFO    MainThread:2196222 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-07 17:18:07,589 INFO    MainThread:2196222 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-07 17:18:07,589 INFO    MainThread:2196222 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-07 17:18:07,589 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-07 17:18:07,589 INFO    HandlerThread:2196222 [handler.py:finish():882] shutting down handler
+2025-01-07 17:18:08,544 INFO    WriterThread:2196222 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/run-6qq3p71c.wandb
+2025-01-07 17:18:08,589 INFO    SenderThread:2196222 [sender.py:finish():1615] shutting down sender
+2025-01-07 17:18:08,589 INFO    SenderThread:2196222 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 17:18:08,589 INFO    SenderThread:2196222 [file_pusher.py:join():175] waiting for file pusher
diff --git a/6qq3p71c/wandb/debug.log b/6qq3p71c/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..fdab9cff2b4aa6b8068a4817d15b9093cb76a6cf
--- /dev/null
+++ b/6qq3p71c/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Configure stats pid to 2196057
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/settings
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/logs/debug.log
+2025-01-07 17:14:04,426 INFO    MainThread:2196057 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/logs/debug-internal.log
+2025-01-07 17:14:04,426 INFO    MainThread:2196057 [wandb_init.py:init():569] calling init triggers
+2025-01-07 17:14:04,426 INFO    MainThread:2196057 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c'}
+2025-01-07 17:14:04,426 INFO    MainThread:2196057 [wandb_init.py:init():619] starting backend
+2025-01-07 17:14:04,426 INFO    MainThread:2196057 [wandb_init.py:init():623] setting up manager
+2025-01-07 17:14:04,430 INFO    MainThread:2196057 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-07 17:14:04,431 INFO    MainThread:2196057 [wandb_init.py:init():631] backend started and connected
+2025-01-07 17:14:04,443 INFO    MainThread:2196057 [wandb_init.py:init():720] updated telemetry
+2025-01-07 17:14:04,453 INFO    MainThread:2196057 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-07 17:14:04,702 INFO    MainThread:2196057 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-07 17:14:04,770 INFO    MainThread:2196057 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-07 17:14:04,770 INFO    MainThread:2196057 [wandb_init.py:init():804] starting run threads in backend
+2025-01-07 17:14:05,123 INFO    MainThread:2196057 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-07 17:14:05,123 INFO    MainThread:2196057 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-07 17:14:05,123 INFO    MainThread:2196057 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-07 17:14:05,124 INFO    MainThread:2196057 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-07 17:14:05,125 INFO    MainThread:2196057 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-07 17:14:05,125 INFO    MainThread:2196057 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/tb, True
+2025-01-07 17:14:07,444 INFO    MainThread:2196057 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2025-01-07 17:18:08,590 WARNING MsgRouterThr:2196057 [router.py:message_loop():77] message_loop has been closed
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code/FISH/eval_robot.py b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d5c842af07e34efefd4f7c65f43f6415d613b902
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 1
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1009_1'
+wandb_notes:
+  desc: null
+  value: 1009_1_restarted_3
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 5
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 1
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1736288044
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: hbm
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/diff.patch b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..c0161c26d9b20c7c85242592d6833c2df9767742
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/diff.patch
@@ -0,0 +1,20 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index a5e82a4..aa5b9b9 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -134,13 +134,13 @@ load_checkpoint: ${agent.load_checkpoint}
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1002_0' 
+-# wandb_run_id: '1009_1' # seed 1
++wandb_run_id: '1009_1' # seed 1
+ # wandb_run_id: '1022_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+ # wandb_run_id: '1003_0'
+ # wandb_run_id: '1013_1' # seed 1
+-wandb_run_id: '1017_0' # dataset shuffle seed 1
++# wandb_run_id: '1017_0' # dataset shuffle seed 1
+ 
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]}
\ No newline at end of file
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..f245b8705b4023daccb20041b22ee7efa342bdc2
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[40, 40, 1.0, 0.9973071514565537, 1, 0.9194368123169305, 1, 0.9303731656703893, 1, 1, 1]]}
\ No newline at end of file
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4 b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9a533ef26f078d4ff471af6ec3b50356e77fdbf3
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:393b812fc042bd7e1762bb31f89641f23d83d9a7bccf663152ad6f2502cdd75c
+size 1128747
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4 b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..7a9d1a2b9053e50da96cd447e157abf8d2469657
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19268ebe421ff04fe7f07cba82a91497bcf023a2d0526110b2982654792f301b
+size 1142439
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4 b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..da947de61c8e19ef8a1466dd2caf38799d64af9c
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d676baf746a2d4745effc19bc6a12a19adb6dfa8742a5b6b0537f33e50c77f52
+size 1148399
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4 b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..3ce05e3ce0935a36ccb73738d0c10e9d62fa6829
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c26e92b3a127f92ec68d16c302727ae990607e8ee5c7ecfaca76e7ea315c3ec
+size 1146191
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4 b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5ce06b5b50a7c37f2db9e060253486930008e777
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d693180a6d9e048e7c1db45a7f0a3fa444e0f4e9a22a930e02f109758873c7f4
+size 1139534
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..9e911d37cbd87d863aa0f12d322fbd252cf7cfd9
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
@@ -0,0 +1,271 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1736288047.446354]: resetting environment
+[INFO] [1736288047.453040]: cleared current plan
+[INFO] [1736288047.454401]: moving to home
+[INFO] [1736288048.456647]: reached home
+[INFO] [1736288048.457992]: reset action history
+[INFO] [1736288051.166052]: environment reset
+Starting episode 0
+[INFO] [1736288051.168266]: resetting environment
+[INFO] [1736288051.170970]: cleared current plan
+[INFO] [1736288051.171934]: moving to home
+[INFO] [1736288052.174192]: reached home
+[INFO] [1736288052.176315]: reset action history
+[INFO] [1736288054.884992]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1736288058.772756]: resetting environment
+[INFO] [1736288058.775386]: cleared current plan
+[INFO] [1736288058.776317]: moving to home
+[INFO] [1736288059.778693]: reached home
+[INFO] [1736288059.780184]: reset action history
+[INFO] [1736288062.488178]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/episode_rosbags/episode_0_2025-01-07-17-14-22.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2196057.log
+started roslaunch server http://158.130.50.37:45693/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2196323]
+process[print_text-2]: started with pid [2196324]
+process[rosbag_record-3]: started with pid [2196348]
+started bagging!
+[INFO] [1736288089.470625]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1736288090.213810]: Stored episode 1.
+[INFO] [1736288090.214220]: Saving video...
+[INFO] [1736288090.407353]: Video saved!
+Starting episode 1
+[INFO] [1736288090.424573]: resetting environment
+[INFO] [1736288090.429065]: cleared current plan
+[INFO] [1736288090.429331]: moving to home
+[INFO] [1736288093.932065]: reached home
+[INFO] [1736288093.932504]: reset action history
+[INFO] [1736288096.639688]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736288101.545175]: resetting environment
+[INFO] [1736288101.547077]: cleared current plan
+[INFO] [1736288101.547243]: moving to home
+[INFO] [1736288102.548676]: reached home
+[INFO] [1736288102.549088]: reset action history
+[INFO] [1736288105.256199]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/episode_rosbags/episode_1_2025-01-07-17-15-05.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2196057.log
+started roslaunch server http://158.130.50.37:34905/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2196456]
+process[print_text-5]: started with pid [2196457]
+process[rosbag_record-6]: started with pid [2196458]
+started bagging!
+For topic gripper_width: timestamp difference is -124921717 for nearest: 1736288108578846536 - target: 1736288108703768253 at idx 51
+For topic gripper_width: timestamp difference is -145909032 for nearest: 1736288123978846112 - target: 1736288124124755144 at idx 249
+[INFO] [1736288132.210189]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1736288132.990908]: Stored episode 2.
+[INFO] [1736288132.991155]: Saving video...
+[INFO] [1736288133.398446]: Video saved!
+Starting episode 2
+[INFO] [1736288133.436094]: resetting environment
+[INFO] [1736288133.446242]: cleared current plan
+[INFO] [1736288133.446503]: moving to home
+[INFO] [1736288136.948453]: reached home
+[INFO] [1736288136.951681]: reset action history
+[INFO] [1736288139.684073]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736288143.883637]: resetting environment
+[INFO] [1736288143.892647]: cleared current plan
+[INFO] [1736288143.892892]: moving to home
+[INFO] [1736288144.897276]: reached home
+[INFO] [1736288144.897884]: reset action history
+[INFO] [1736288147.607549]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/episode_rosbags/episode_2_2025-01-07-17-15-47.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2196057.log
+started roslaunch server http://158.130.50.37:42555/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2196627]
+process[print_text-8]: started with pid [2196628]
+process[rosbag_record-9]: started with pid [2196629]
+started bagging!
+For topic gripper_width: timestamp difference is -114301312 for nearest: 1736288163078846109 - target: 1736288163193147421 at idx 214
+[INFO] [1736288174.653705]: Storing episode...
+[rosbag_record-9] killing on exit
+[pub_text-7] killing on exit
+[print_text-8] killing on exit
+[INFO] [1736288175.483296]: Stored episode 3.
+[INFO] [1736288175.487671]: Saving video...
+[INFO] [1736288175.928281]: Video saved!
+Starting episode 3
+[INFO] [1736288175.963236]: resetting environment
+[INFO] [1736288175.979364]: cleared current plan
+[INFO] [1736288175.982054]: moving to home
+[WARN] [1736288175.816678]: Plan exhausted
+[WARN] [1736288175.855680]: Plan exhausted
+[WARN] [1736288175.895124]: Plan exhausted
+[WARN] [1736288175.936891]: Plan exhausted
+[WARN] [1736288175.975119]: Plan exhausted
+[INFO] [1736288179.488671]: reached home
+[INFO] [1736288179.490201]: reset action history
+[INFO] [1736288182.271283]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736288189.497045]: resetting environment
+[INFO] [1736288189.511681]: cleared current plan
+[INFO] [1736288189.513828]: moving to home
+[INFO] [1736288190.518853]: reached home
+[INFO] [1736288190.521546]: reset action history
+[INFO] [1736288193.395998]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/episode_rosbags/episode_3_2025-01-07-17-16-33.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2196057.log
+started roslaunch server http://158.130.50.37:42313/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2196763]
+process[print_text-11]: started with pid [2196764]
+process[rosbag_record-12]: started with pid [2196765]
+started bagging!
+[INFO] [1736288220.519458]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit[pub_text-10] killing on exit
+[INFO] [1736288221.315220]: Stored episode 4.
+[INFO] [1736288221.323039]: Saving video...
+[INFO] [1736288221.611912]: Video saved!
+Starting episode 4
+[INFO] [1736288221.667188]: resetting environment
+[INFO] [1736288221.679914]: cleared current plan
+[INFO] [1736288221.680210]: moving to home
+[INFO] [1736288225.185673]: reached home
+[INFO] [1736288225.192642]: reset action history
+[INFO] [1736288227.990526]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736288229.514098]: resetting environment
+[INFO] [1736288229.529114]: cleared current plan
+[INFO] [1736288229.532282]: moving to home
+[INFO] [1736288230.540100]: reached home
+[INFO] [1736288230.546015]: reset action history
+[INFO] [1736288233.321995]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/episode_rosbags/episode_4_2025-01-07-17-17-13.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2196057.log
+started roslaunch server http://158.130.50.37:44333/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2196891]
+process[print_text-14]: started with pid [2196892]
+process[rosbag_record-15]: started with pid [2196893]
+started bagging!
+[INFO] [1736288260.270188]: Storing episode...
+[print_text-14] killing on exit[rosbag_record-15] killing on exit[pub_text-13] killing on exit
+[INFO] [1736288261.100069]: Stored episode 5.
+[INFO] [1736288261.101015]: Saving video...
+[INFO] [1736288261.769383]: Video saved!
+[INFO] [1736288261.833899]: resetting environment
+[INFO] [1736288261.849791]: cleared current plan
+[INFO] [1736288261.851942]: moving to home
+[WARN] [1736288261.625256]: Plan exhausted
+[WARN] [1736288261.671281]: Plan exhausted
+[WARN] [1736288261.706568]: Plan exhausted
+[WARN] [1736288261.747101]: Plan exhausted
+[WARN] [1736288261.786364]: Plan exhausted
+[WARN] [1736288261.827991]: Plan exhausted
+[INFO] [1736288265.354731]: reached home
+[INFO] [1736288265.360890]: reset action history
+[INFO] [1736288268.198443]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/requirements.txt b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-metadata.json b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..8c0462511b336bad0dd96c84809a4e97a4d35c17
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-07T22:14:04.794650",
+    "startedAt": "2025-01-07T22:14:04.419165",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "29237eae507484d4aff7683bad40a564046255b9"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 4124.5576249999995,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4497.613,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.631,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.003,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 743.5787963867188
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..7dc9a14664ea0e97e01bda4657d1a604e0a320ae
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "393b812fc042bd7e1762bb31f89641f23d83d9a7bccf663152ad6f2502cdd75c", "size": 1128747, "path": "media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4"}, "global_step": 664, "_timestamp": 1736288272.5429602, "_runtime": 228.11127424240112, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "19268ebe421ff04fe7f07cba82a91497bcf023a2d0526110b2982654792f301b", "size": 1142439, "path": "media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4"}, "eval/num_success": 5.0, "episode": 40.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "d676baf746a2d4745effc19bc6a12a19adb6dfa8742a5b6b0537f33e50c77f52", "size": 1148399, "path": "media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "0c26e92b3a127f92ec68d16c302727ae990607e8ee5c7ecfaca76e7ea315c3ec", "size": 1146191, "path": "media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d693180a6d9e048e7c1db45a7f0a3fa444e0f4e9a22a930e02f109758873c7f4", "size": 1139534, "path": "media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://saskwwu925m34q9ssiefm2xjzdm8s31zl450l08of9kzc1ng1t53q3vls1410qp8uv2myitfos6azr5yyk02lhttfhmufqg5ryxjcem53or8bgeq8vz0pqlj2vwq9s30/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://526qaby7oeg7zwc12zc6fam53mnv3grvvnld31yrsi06g2iyr9ku9543cksasrghjjg7d4h8dwb3l5ry7fu0ywx3qlh544jfm203x9zf7tumg7qleolzqwt3zw6nkwn8:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 40, "total_num_episodes": 40, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9973071514565537, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.9194368123169305, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.9303731656703893, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "32f8ff8bc7debb3475b0dd7e1f4ada326027eedcc6a834033025848a13497fbb", "size": 420, "artifact_path": "wandb-client-artifact://hl5viu64bus0hs6dndwq7cv43r56mw7upimgg8p7hyp515231vn8hwfoiczqyojhp8hs4v5lgz1m6sma70kerrqf4m3tb8o6lmquwf8ivdu6jbyq63yk4wrn84nfdotw/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vgg45c3xp3a8jklfoxw84qdwnmh5ht9zge9mgn89shurbwt53affudxuvn4jexmxtcsgz2vd44nwcncmkijv4hsmysyog3f25wrc2yaotnlqahzdzew8wlwbbknhqaqd:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 233}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9973071217536926, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.9194368124008179, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9303731918334961, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0}
\ No newline at end of file
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/logs/debug-internal.log b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..64fb7bfe5869f22ea7cba83b21d9af77d392dc28
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/logs/debug-internal.log
@@ -0,0 +1,716 @@
+2025-01-07 17:14:04,431 INFO    StreamThr :2196222 [internal.py:wandb_internal():85] W&B internal server running at pid: 2196222, started at: 2025-01-07 17:14:04.431363
+2025-01-07 17:14:04,432 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status
+2025-01-07 17:14:04,443 INFO    WriterThread:2196222 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/run-6qq3p71c.wandb
+2025-01-07 17:14:04,444 DEBUG   SenderThread:2196222 [sender.py:send():379] send: header
+2025-01-07 17:14:04,455 DEBUG   SenderThread:2196222 [sender.py:send():379] send: run
+2025-01-07 17:14:04,686 INFO    SenderThread:2196222 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files
+2025-01-07 17:14:04,687 INFO    SenderThread:2196222 [sender.py:_start_run_threads():1188] run started: 6qq3p71c with start time 1736288044.431686
+2025-01-07 17:14:04,702 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: check_version
+2025-01-07 17:14:04,702 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: check_version
+2025-01-07 17:14:04,776 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: run_start
+2025-01-07 17:14:04,780 DEBUG   HandlerThread:2196222 [system_info.py:__init__():26] System info init
+2025-01-07 17:14:04,780 DEBUG   HandlerThread:2196222 [system_info.py:__init__():41] System info init done
+2025-01-07 17:14:04,780 INFO    HandlerThread:2196222 [system_monitor.py:start():194] Starting system monitor
+2025-01-07 17:14:04,781 INFO    SystemMonitor:2196222 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-07 17:14:04,781 INFO    HandlerThread:2196222 [system_monitor.py:probe():214] Collecting system info
+2025-01-07 17:14:04,781 INFO    SystemMonitor:2196222 [interfaces.py:start():188] Started cpu monitoring
+2025-01-07 17:14:04,782 INFO    SystemMonitor:2196222 [interfaces.py:start():188] Started disk monitoring
+2025-01-07 17:14:04,783 INFO    SystemMonitor:2196222 [interfaces.py:start():188] Started gpu monitoring
+2025-01-07 17:14:04,783 INFO    SystemMonitor:2196222 [interfaces.py:start():188] Started memory monitoring
+2025-01-07 17:14:04,784 INFO    SystemMonitor:2196222 [interfaces.py:start():188] Started network monitoring
+2025-01-07 17:14:04,794 DEBUG   HandlerThread:2196222 [system_info.py:probe():152] Probing system
+2025-01-07 17:14:04,796 DEBUG   HandlerThread:2196222 [system_info.py:_probe_git():137] Probing git
+2025-01-07 17:14:04,800 DEBUG   HandlerThread:2196222 [system_info.py:_probe_git():145] Probing git done
+2025-01-07 17:14:04,800 DEBUG   HandlerThread:2196222 [system_info.py:probe():200] Probing system done
+2025-01-07 17:14:04,800 DEBUG   HandlerThread:2196222 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:14:04.794650', 'startedAt': '2025-01-07T22:14:04.419165', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.5576249999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.613, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.631, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 743.5787963867188}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-07 17:14:04,800 INFO    HandlerThread:2196222 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-07 17:14:04,801 INFO    HandlerThread:2196222 [system_monitor.py:probe():227] Publishing system info
+2025-01-07 17:14:04,801 DEBUG   HandlerThread:2196222 [system_info.py:_save_code():44] Saving code
+2025-01-07 17:14:04,806 DEBUG   HandlerThread:2196222 [system_info.py:_save_code():67] Saving code done
+2025-01-07 17:14:04,806 DEBUG   HandlerThread:2196222 [system_info.py:_save_patches():84] Saving git patches
+2025-01-07 17:14:04,876 DEBUG   HandlerThread:2196222 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-07 17:14:04,877 INFO    HandlerThread:2196222 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-07 17:14:04,880 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:14:04,880 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-07 17:14:04,880 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-07 17:14:04,880 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-07 17:14:05,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-07 17:14:05,123 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: python_packages
+2025-01-07 17:14:05,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:14:05,123 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:14:05,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:05,126 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/pm4k7w63-wandb-metadata.json
+2025-01-07 17:14:05,126 INFO    HandlerThread:2196222 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/tb"
+  save: true
+}
+control {
+  relay_id: "38255728e4c045ed8f1dd189a41437c0"
+}
+_info {
+  stream_id: "6qq3p71c"
+}
+
+2025-01-07 17:14:05,126 DEBUG   HandlerThread:2196222 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-07 17:14:05,178 INFO    wandb-upload_1:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/nl9lvu9s-code/FISH/eval_robot.py
+2025-01-07 17:14:05,187 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/kbip47ap-diff.patch
+2025-01-07 17:14:05,239 DEBUG   SenderThread:2196222 [sender.py:send():379] send: telemetry
+2025-01-07 17:14:05,239 DEBUG   SenderThread:2196222 [sender.py:send():379] send: tbrecord
+2025-01-07 17:14:05,240 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:14:05,240 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0 with policy live
+2025-01-07 17:14:05,432 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/crfyvop6-events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:14:05,688 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code/FISH/eval_robot.py
+2025-01-07 17:14:05,688 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-metadata.json
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/requirements.txt
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/diff.patch
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code/FISH
+2025-01-07 17:14:05,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code
+2025-01-07 17:14:06,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:07,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:07,445 DEBUG   SenderThread:2196222 [sender.py:send():379] send: config
+2025-01-07 17:14:07,688 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:08,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:09,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:09,497 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:09,688 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:10,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:11,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:12,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:13,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:13,689 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:14,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:14,885 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:15,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:16,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:17,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:17,690 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:18,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:19,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:20,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:14:20,123 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:14:20,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:20,250 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:21,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:21,692 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:22,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:23,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:23,692 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:24,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:25,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:25,693 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:26,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:26,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:27,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:28,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:29,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:30,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:31,124 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:31,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:32,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:33,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:34,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:35,123 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:14:35,123 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:14:35,165 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:36,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:36,225 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:36,696 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml
+2025-01-07 17:14:37,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:38,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:39,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:40,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:41,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:42,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:42,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:43,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:44,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:45,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:46,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:47,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:48,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:48,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:49,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:50,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:14:50,125 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:14:50,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:50,423 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:14:50,423 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4 with policy now
+2025-01-07 17:14:50,423 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:14:50,424 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:14:50,424 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:14:50,426 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:14:50,673 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/wmm0r09y-media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4
+2025-01-07 17:14:50,699 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:14:50,700 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4
+2025-01-07 17:14:50,700 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media
+2025-01-07 17:14:50,700 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval
+2025-01-07 17:14:50,701 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos
+2025-01-07 17:14:50,968 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/n6l1p8xt-media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4
+2025-01-07 17:14:51,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:51,700 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:52,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:53,130 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:53,430 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:54,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:55,130 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:55,701 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:56,130 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:57,130 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:57,702 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:14:58,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:14:58,681 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:14:59,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:00,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:01,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:01,703 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:15:02,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:03,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:03,704 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:04,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:04,593 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:04,784 DEBUG   SystemMonitor:2196222 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-07 17:15:04,785 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:15:05,127 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:15:05,127 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:15:05,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:06,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:07,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:07,705 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:08,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:09,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:09,861 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:10,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:11,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:11,706 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:12,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:13,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:14,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:15,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:15,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:16,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:17,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:18,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:19,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:20,125 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:15:20,125 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:15:20,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:20,239 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:21,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:22,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:23,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:24,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:25,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:25,711 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:26,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:26,134 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:27,134 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:28,134 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:29,134 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:30,135 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:31,136 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:32,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:32,136 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:33,136 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:33,445 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:15:33,445 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:15:33,445 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4 with policy now
+2025-01-07 17:15:33,446 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:15:33,446 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:15:33,447 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:15:33,706 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/zg8yj1wa-media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4
+2025-01-07 17:15:33,713 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:33,714 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:15:33,714 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4
+2025-01-07 17:15:33,714 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval
+2025-01-07 17:15:33,980 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/3n0ami7s-media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4
+2025-01-07 17:15:34,138 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:34,785 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:15:35,130 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:15:35,130 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:15:35,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:35,714 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:36,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:37,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:37,993 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:38,144 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:39,143 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:39,715 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:40,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:41,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:41,716 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:42,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:43,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:43,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:44,140 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:44,518 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:15:44,518 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:15:44,519 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:15:44,717 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:15:44,717 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:15:44,978 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/pzupy8xk-events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:15:45,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:45,718 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:46,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:47,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:47,718 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:48,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:48,953 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:49,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:49,719 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:15:50,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:15:50,126 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:15:50,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:51,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:52,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:53,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:54,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:54,293 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:15:55,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:56,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:57,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:58,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:15:59,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:00,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:00,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:01,142 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:02,144 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:03,143 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:04,144 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:04,786 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:16:05,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:16:05,126 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:16:05,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:05,222 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:05,725 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:06,144 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:07,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:08,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:09,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:10,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:11,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:11,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:12,145 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:13,146 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:14,146 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:15,149 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:15,728 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:15,950 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:16:15,950 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4 with policy now
+2025-01-07 17:16:15,958 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:16:15,958 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:16:15,958 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:16:15,959 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:16:16,150 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:16,202 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/4er4aazk-media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4
+2025-01-07 17:16:16,728 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:16:16,728 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4
+2025-01-07 17:16:16,728 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval
+2025-01-07 17:16:16,982 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:17,009 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/3i7cr54k-media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4
+2025-01-07 17:16:17,151 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:17,729 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:18,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:18,729 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:19,150 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:20,129 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:16:20,129 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:16:20,152 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:21,154 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:21,731 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:22,152 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:22,272 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:23,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:23,731 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:24,156 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:25,155 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:26,157 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:27,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:28,131 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:28,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:29,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:29,733 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:16:30,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:30,354 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:16:30,354 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:16:30,355 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:16:30,734 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:16:31,158 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:31,734 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:32,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:33,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:33,396 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:34,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:34,786 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:16:35,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:16:35,126 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:16:35,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:35,735 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:16:36,158 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:37,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:38,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:39,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:39,215 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:40,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:41,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:42,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:43,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:44,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:45,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:45,159 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:46,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:47,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:48,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:49,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:50,126 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:16:50,126 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:16:50,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:50,291 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:51,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:52,160 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:53,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:54,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:55,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:56,132 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:16:56,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:57,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:58,161 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:16:59,164 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:00,164 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:01,165 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:01,317 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:01,661 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:17:01,661 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4 with policy now
+2025-01-07 17:17:01,662 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:17:01,663 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:01,663 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:01,663 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:01,744 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:01,744 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:01,744 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4
+2025-01-07 17:17:01,745 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval
+2025-01-07 17:17:01,922 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/qf3aacrc-media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4
+2025-01-07 17:17:02,164 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:02,189 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/pjenyrtw-media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4
+2025-01-07 17:17:03,165 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:03,745 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:04,167 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:04,787 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:17:05,138 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:17:05,138 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:17:05,185 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:06,167 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:07,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:07,259 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:07,746 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:08,167 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:09,168 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:09,747 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:09,747 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:17:10,000 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/d6zk95u6-events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:17:10,168 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:10,406 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:10,406 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:10,406 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:10,747 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:11,171 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:11,748 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:12,171 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:12,585 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:13,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:14,168 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:15,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:15,749 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:16,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:17,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:18,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:18,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:19,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:20,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:17:20,128 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:17:20,169 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:21,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:22,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:23,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:23,224 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:24,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:25,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:26,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:27,170 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:28,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:29,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:29,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:30,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:31,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:32,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:33,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:34,133 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:34,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:34,787 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:17:35,128 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:17:35,128 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:17:35,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:36,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:37,172 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:38,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:39,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:39,222 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:40,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:41,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:41,758 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:41,822 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:17:41,822 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4 with policy now
+2025-01-07 17:17:41,827 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:17:41,828 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:41,828 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:41,828 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:42,078 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/32uoeb2q-media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4
+2025-01-07 17:17:42,173 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:42,758 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:42,758 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4
+2025-01-07 17:17:42,758 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval
+2025-01-07 17:17:43,070 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/s3n9j6oy-media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4
+2025-01-07 17:17:43,175 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:43,758 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:44,181 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:44,759 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:44,852 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:45,180 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:46,178 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:47,182 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:47,760 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:48,186 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:49,186 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:49,761 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:50,144 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:50,146 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:17:50,146 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:17:50,190 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:51,185 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:51,762 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:17:52,081 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 17:17:52,082 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 17:17:52,183 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:52,262 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:17:52,601 INFO    wandb-upload_2:2196222 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_hat441t
+2025-01-07 17:17:52,635 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 17:17:52,643 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:17:52,762 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:17:52,762 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:17:52,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json
+2025-01-07 17:17:52,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media
+2025-01-07 17:17:52,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval
+2025-01-07 17:17:52,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table
+2025-01-07 17:17:53,001 INFO    SenderThread:2196222 [sender.py:send_request_log_artifact():1525] logged artifact run-6qq3p71c-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkwOTQ1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY5NjEwOQ==', 'latestArtifact': None}}
+2025-01-07 17:17:53,001 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:17:53,001 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now
+2025-01-07 17:17:53,001 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:53,001 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,001 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,002 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:53,002 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,002 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,003 DEBUG   SenderThread:2196222 [sender.py:send():379] send: telemetry
+2025-01-07 17:17:53,003 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,003 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,003 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,003 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,004 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,004 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,004 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,004 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,005 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,005 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,005 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,005 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,005 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,006 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,006 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,006 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,006 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,007 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,007 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,007 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,007 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,007 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:53,008 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 17:17:53,177 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/6c7xiqr1-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:17:53,181 INFO    wandb-upload_1:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/wq18ni64-media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json
+2025-01-07 17:17:53,183 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:53,417 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/p7744o24-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:17:53,482 INFO    wandb-upload_2:2196222 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppgxspz31
+2025-01-07 17:17:53,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:17:53,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:53,842 INFO    SenderThread:2196222 [sender.py:send_request_log_artifact():1525] logged artifact run-6qq3p71c-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTkwOTQ5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY5NjExMQ==', 'latestArtifact': None}}
+2025-01-07 17:17:53,842 DEBUG   SenderThread:2196222 [sender.py:send():379] send: files
+2025-01-07 17:17:53,842 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json with policy now
+2025-01-07 17:17:53,843 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:17:53,843 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:17:53,843 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:54,078 INFO    wandb-upload_1:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/hyz89z7w-media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json
+2025-01-07 17:17:54,184 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:54,763 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:55,184 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:56,141 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:17:56,185 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:56,764 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml
+2025-01-07 17:17:57,185 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:58,185 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:17:58,686 DEBUG   SenderThread:2196222 [sender.py:send():379] send: exit
+2025-01-07 17:17:58,686 INFO    SenderThread:2196222 [sender.py:send_exit():586] handling exit code: 0
+2025-01-07 17:17:58,686 INFO    SenderThread:2196222 [sender.py:send_exit():588] handling runtime: 233
+2025-01-07 17:17:58,686 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:17:58,686 INFO    SenderThread:2196222 [sender.py:send_exit():594] send defer
+2025-01-07 17:17:58,686 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:17:58,686 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-07 17:17:58,687 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:17:58,687 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-07 17:17:58,687 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 1
+2025-01-07 17:17:58,687 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:17:58,687 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-07 17:17:58,687 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:17:58,687 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-07 17:17:58,687 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 2
+2025-01-07 17:17:58,687 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:17:58,687 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-07 17:17:58,688 INFO    HandlerThread:2196222 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-07 17:17:58,688 DEBUG   SystemMonitor:2196222 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-07 17:17:58,688 DEBUG   SystemMonitor:2196222 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-07 17:17:58,688 INFO    HandlerThread:2196222 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-07 17:17:58,689 INFO    HandlerThread:2196222 [interfaces.py:finish():200] Joined disk monitor
+2025-01-07 17:17:58,740 INFO    HandlerThread:2196222 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-07 17:17:58,741 INFO    HandlerThread:2196222 [interfaces.py:finish():200] Joined memory monitor
+2025-01-07 17:17:58,741 INFO    HandlerThread:2196222 [interfaces.py:finish():200] Joined network monitor
+2025-01-07 17:17:58,741 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:17:58,741 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-07 17:17:58,741 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 3
+2025-01-07 17:17:58,741 DEBUG   SenderThread:2196222 [sender.py:send():379] send: stats
+2025-01-07 17:17:58,742 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:17:58,742 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-07 17:17:58,742 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:17:58,742 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-07 17:17:58,742 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 4
+2025-01-07 17:17:58,742 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:17:58,742 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-07 17:17:58,765 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:17:59,765 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:18:05,460 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:18:05,461 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:18:05,461 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-07 17:18:05,462 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:05,462 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-07 17:18:05,462 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 5
+2025-01-07 17:18:05,462 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:05,462 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:18:05,462 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-07 17:18:05,462 DEBUG   SenderThread:2196222 [sender.py:send():379] send: history
+2025-01-07 17:18:05,463 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:18:05,464 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:18:05,464 DEBUG   SenderThread:2196222 [sender.py:send():379] send: summary
+2025-01-07 17:18:05,468 INFO    SenderThread:2196222 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:18:05,468 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:05,468 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-07 17:18:05,468 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 6
+2025-01-07 17:18:05,468 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:05,468 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-07 17:18:05,469 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:05,469 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-07 17:18:05,469 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 7
+2025-01-07 17:18:05,469 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:18:05,469 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:05,469 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-07 17:18:05,469 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:05,469 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-07 17:18:05,687 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:18:05,768 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:18:05,928 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 8
+2025-01-07 17:18:05,928 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:18:05,928 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:05,928 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-07 17:18:05,928 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:05,928 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-07 17:18:05,928 INFO    SenderThread:2196222 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-07 17:18:05,931 INFO    SenderThread:2196222 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-07 17:18:05,933 INFO    SenderThread:2196222 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-07 17:18:05,946 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 9
+2025-01-07 17:18:05,946 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:05,946 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-07 17:18:05,946 DEBUG   SenderThread:2196222 [sender.py:send():379] send: artifact
+2025-01-07 17:18:06,151 INFO    SenderThread:2196222 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'versionIndex': 435}}}
+2025-01-07 17:18:06,151 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:06,151 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-07 17:18:06,151 INFO    SenderThread:2196222 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-07 17:18:06,687 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:18:06,768 INFO    Thread-12 :2196222 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml config.yaml
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json wandb-summary.json
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/requirements.txt requirements.txt
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-metadata.json wandb-metadata.json
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0 events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log output.log
+2025-01-07 17:18:06,769 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/diff.patch diff.patch
+2025-01-07 17:18:06,774 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-07 17:18:06,776 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4 media/videos/eval/2_eval_2_d676baf746a2d4745eff.mp4
+2025-01-07 17:18:06,776 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4 media/videos/eval/3_eval_3_0c26e92b3a127f92ec68.mp4
+2025-01-07 17:18:06,776 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4 media/videos/eval/1_eval_1_19268ebe421ff04fe7f0.mp4
+2025-01-07 17:18:06,776 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4 media/videos/eval/4_eval_4_d693180a6d9e048e7c1d.mp4
+2025-01-07 17:18:06,777 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4 media/videos/eval/0_eval_0_393b812fc042bd7e1762.mp4
+2025-01-07 17:18:06,778 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json media/table/eval/total_success_rate_ci_6_32f8ff8bc7debb3475b0.table.json
+2025-01-07 17:18:06,778 INFO    SenderThread:2196222 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:18:06,778 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 10
+2025-01-07 17:18:06,778 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:18:06,778 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:06,779 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-07 17:18:06,779 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:06,779 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-07 17:18:06,779 INFO    SenderThread:2196222 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 17:18:06,951 INFO    wandb-upload_1:2196222 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/requirements.txt
+2025-01-07 17:18:06,952 INFO    wandb-upload_2:2196222 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/config.yaml
+2025-01-07 17:18:06,984 INFO    wandb-upload_4:2196222 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/output.log
+2025-01-07 17:18:07,000 INFO    wandb-upload_0:2196222 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/files/wandb-summary.json
+2025-01-07 17:18:07,045 INFO    wandb-upload_3:2196222 [upload_job.py:push():130] Uploaded file /tmp/tmpqcd60463wandb/urz358fq-events.out.tfevents.1736288045.leonmkim-ROG-Strix-G15CS-G15CS.2196057.0
+2025-01-07 17:18:07,246 INFO    Thread-11 (_thread_body):2196222 [sender.py:transition_state():614] send defer: 11
+2025-01-07 17:18:07,246 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:07,246 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-07 17:18:07,247 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:07,247 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-07 17:18:07,247 INFO    SenderThread:2196222 [file_pusher.py:join():175] waiting for file pusher
+2025-01-07 17:18:07,248 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 12
+2025-01-07 17:18:07,248 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:07,248 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-07 17:18:07,248 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:07,248 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-07 17:18:07,248 INFO    SenderThread:2196222 [file_stream.py:finish():601] file stream finish called
+2025-01-07 17:18:07,538 INFO    SenderThread:2196222 [file_stream.py:finish():605] file stream finish is done
+2025-01-07 17:18:07,538 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 13
+2025-01-07 17:18:07,538 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:07,538 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-07 17:18:07,538 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:07,538 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-07 17:18:07,539 INFO    SenderThread:2196222 [sender.py:transition_state():614] send defer: 14
+2025-01-07 17:18:07,539 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:18:07,539 INFO    HandlerThread:2196222 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-07 17:18:07,539 DEBUG   SenderThread:2196222 [sender.py:send():379] send: final
+2025-01-07 17:18:07,539 DEBUG   SenderThread:2196222 [sender.py:send():379] send: footer
+2025-01-07 17:18:07,539 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:18:07,539 INFO    SenderThread:2196222 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-07 17:18:07,539 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:18:07,540 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:18:07,540 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: server_info
+2025-01-07 17:18:07,540 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: server_info
+2025-01-07 17:18:07,543 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:18:07,543 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-07 17:18:07,544 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-07 17:18:07,545 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:18:07,588 DEBUG   SenderThread:2196222 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:18:07,589 INFO    MainThread:2196222 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-07 17:18:07,589 INFO    MainThread:2196222 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-07 17:18:07,589 INFO    MainThread:2196222 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-07 17:18:07,589 DEBUG   HandlerThread:2196222 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-07 17:18:07,589 INFO    HandlerThread:2196222 [handler.py:finish():882] shutting down handler
+2025-01-07 17:18:08,544 INFO    WriterThread:2196222 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/run-6qq3p71c.wandb
+2025-01-07 17:18:08,589 INFO    SenderThread:2196222 [sender.py:finish():1615] shutting down sender
+2025-01-07 17:18:08,589 INFO    SenderThread:2196222 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 17:18:08,589 INFO    SenderThread:2196222 [file_pusher.py:join():175] waiting for file pusher
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/logs/debug.log b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..fdab9cff2b4aa6b8068a4817d15b9093cb76a6cf
--- /dev/null
+++ b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Configure stats pid to 2196057
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/settings
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-07 17:14:04,425 INFO    MainThread:2196057 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/logs/debug.log
+2025-01-07 17:14:04,426 INFO    MainThread:2196057 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/logs/debug-internal.log
+2025-01-07 17:14:04,426 INFO    MainThread:2196057 [wandb_init.py:init():569] calling init triggers
+2025-01-07 17:14:04,426 INFO    MainThread:2196057 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c'}
+2025-01-07 17:14:04,426 INFO    MainThread:2196057 [wandb_init.py:init():619] starting backend
+2025-01-07 17:14:04,426 INFO    MainThread:2196057 [wandb_init.py:init():623] setting up manager
+2025-01-07 17:14:04,430 INFO    MainThread:2196057 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-07 17:14:04,431 INFO    MainThread:2196057 [wandb_init.py:init():631] backend started and connected
+2025-01-07 17:14:04,443 INFO    MainThread:2196057 [wandb_init.py:init():720] updated telemetry
+2025-01-07 17:14:04,453 INFO    MainThread:2196057 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-07 17:14:04,702 INFO    MainThread:2196057 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-07 17:14:04,770 INFO    MainThread:2196057 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-07 17:14:04,770 INFO    MainThread:2196057 [wandb_init.py:init():804] starting run threads in backend
+2025-01-07 17:14:05,123 INFO    MainThread:2196057 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-07 17:14:05,123 INFO    MainThread:2196057 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-07 17:14:05,123 INFO    MainThread:2196057 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-07 17:14:05,124 INFO    MainThread:2196057 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-07 17:14:05,125 INFO    MainThread:2196057 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-07 17:14:05,125 INFO    MainThread:2196057 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/6qq3p71c/tb, True
+2025-01-07 17:14:07,444 INFO    MainThread:2196057 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2025-01-07 17:18:08,590 WARNING MsgRouterThr:2196057 [router.py:message_loop():77] message_loop has been closed
diff --git a/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/run-6qq3p71c.wandb b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/run-6qq3p71c.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..e556038108cb198fd8f9b2ccf241e4390e33109f
Binary files /dev/null and b/6qq3p71c/wandb/run-20250107_171404-6qq3p71c/run-6qq3p71c.wandb differ
diff --git a/9fviy2t8/.hydra/config.yaml b/9fviy2t8/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab
--- /dev/null
+++ b/9fviy2t8/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1009_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/9fviy2t8/.hydra/hydra.yaml b/9fviy2t8/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d2f19621d9305ecf4d1c42a7b7ce31b75a533152
--- /dev/null
+++ b/9fviy2t8/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/9fviy2t8/.hydra/overrides.yaml b/9fviy2t8/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/9fviy2t8/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/9fviy2t8/episode_rosbags/aligned_depth_to_color_K.npy b/9fviy2t8/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/9fviy2t8/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/9fviy2t8/episode_rosbags/cam_tf_world.npy b/9fviy2t8/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/9fviy2t8/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/9fviy2t8/episode_rosbags/color_K.npy b/9fviy2t8/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/9fviy2t8/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/9fviy2t8/episode_rosbags/depth_K.npy b/9fviy2t8/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/9fviy2t8/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/9fviy2t8/eval_robot.log b/9fviy2t8/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..47f5c687169e4598919cadf79c5a024af47bae66
--- /dev/null
+++ b/9fviy2t8/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-06 17:23:31,994][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-06 17:23:31,997][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/9fviy2t8/eval_video/0_eval.mp4 b/9fviy2t8/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..7478195d9e30f7cc4ef1f2e7138fe97ba8a0add6
--- /dev/null
+++ b/9fviy2t8/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fd61fcba1e69d329c7453ca0f5c4e8c027634331e07ca6e286da68c41227226
+size 1142388
diff --git a/9fviy2t8/eval_video/1_eval.mp4 b/9fviy2t8/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..a3d0b0c415badc728f2af8bc300ae2fbab93e996
--- /dev/null
+++ b/9fviy2t8/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23c6d820f71940e1e98dc187f36572a14dcf37f4660569737a14ae95fe3fd579
+size 1138983
diff --git a/9fviy2t8/eval_video/2_eval.mp4 b/9fviy2t8/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..952819566e460d624dc17d5c35d7c74e0a64c855
--- /dev/null
+++ b/9fviy2t8/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bda5f29eb6d3686af62f4e6a91088d3a306986aaf7ed02443d20bc532e459e3
+size 1151555
diff --git a/9fviy2t8/eval_video/3_eval.mp4 b/9fviy2t8/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9511eeaf42883177445230d2e63aacda8a133212
--- /dev/null
+++ b/9fviy2t8/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f833856d563f00bfb38f835f8817b79977c979df6bc732bc53c7881cd00ff48
+size 1163215
diff --git a/9fviy2t8/eval_video/4_eval.mp4 b/9fviy2t8/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..be99ccdd7a9535eb2a96f397988bea42e0357a50
--- /dev/null
+++ b/9fviy2t8/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a47514f65ac4ebdff905da925320694cb5b8cbad4e6f5d8c4bd25194bc310d9b
+size 1140822
diff --git a/9fviy2t8/tb/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0 b/9fviy2t8/tb/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
new file mode 100644
index 0000000000000000000000000000000000000000..d9423144ff904c1d04b7cf86112cf9bc62b26e7d
--- /dev/null
+++ b/9fviy2t8/tb/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3142a7f38b1de3dd7b89d66da1f59c369c2ee267b7a4f682b244ecf396e3da6
+size 6539
diff --git a/9fviy2t8/wandb/debug-internal.log b/9fviy2t8/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..9a119ebb0219ae8cb50396dab79f019389d0ad52
--- /dev/null
+++ b/9fviy2t8/wandb/debug-internal.log
@@ -0,0 +1,763 @@
+2025-01-06 17:23:37,744 INFO    StreamThr :2089241 [internal.py:wandb_internal():85] W&B internal server running at pid: 2089241, started at: 2025-01-06 17:23:37.744072
+2025-01-06 17:23:37,745 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status
+2025-01-06 17:23:37,760 INFO    WriterThread:2089241 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/run-9fviy2t8.wandb
+2025-01-06 17:23:37,760 DEBUG   SenderThread:2089241 [sender.py:send():379] send: header
+2025-01-06 17:23:37,768 DEBUG   SenderThread:2089241 [sender.py:send():379] send: run
+2025-01-06 17:23:38,176 INFO    SenderThread:2089241 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files
+2025-01-06 17:23:38,176 INFO    SenderThread:2089241 [sender.py:_start_run_threads():1188] run started: 9fviy2t8 with start time 1736202217.744472
+2025-01-06 17:23:38,193 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: check_version
+2025-01-06 17:23:38,194 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: check_version
+2025-01-06 17:23:38,270 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: run_start
+2025-01-06 17:23:38,275 DEBUG   HandlerThread:2089241 [system_info.py:__init__():26] System info init
+2025-01-06 17:23:38,275 DEBUG   HandlerThread:2089241 [system_info.py:__init__():41] System info init done
+2025-01-06 17:23:38,275 INFO    HandlerThread:2089241 [system_monitor.py:start():194] Starting system monitor
+2025-01-06 17:23:38,275 INFO    SystemMonitor:2089241 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-06 17:23:38,275 INFO    SystemMonitor:2089241 [interfaces.py:start():188] Started cpu monitoring
+2025-01-06 17:23:38,276 INFO    HandlerThread:2089241 [system_monitor.py:probe():214] Collecting system info
+2025-01-06 17:23:38,276 INFO    SystemMonitor:2089241 [interfaces.py:start():188] Started disk monitoring
+2025-01-06 17:23:38,277 INFO    SystemMonitor:2089241 [interfaces.py:start():188] Started gpu monitoring
+2025-01-06 17:23:38,277 INFO    SystemMonitor:2089241 [interfaces.py:start():188] Started memory monitoring
+2025-01-06 17:23:38,278 INFO    SystemMonitor:2089241 [interfaces.py:start():188] Started network monitoring
+2025-01-06 17:23:38,288 DEBUG   HandlerThread:2089241 [system_info.py:probe():152] Probing system
+2025-01-06 17:23:38,289 DEBUG   HandlerThread:2089241 [system_info.py:_probe_git():137] Probing git
+2025-01-06 17:23:38,294 DEBUG   HandlerThread:2089241 [system_info.py:_probe_git():145] Probing git done
+2025-01-06 17:23:38,294 DEBUG   HandlerThread:2089241 [system_info.py:probe():200] Probing system done
+2025-01-06 17:23:38,294 DEBUG   HandlerThread:2089241 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:23:38.288383', 'startedAt': '2025-01-06T22:23:37.737194', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.00175, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.983, 'min': 800.0, 'max': 4700.0}, {'current': 4492.017, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.312, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 334.0262336730957}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-06 17:23:38,294 INFO    HandlerThread:2089241 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-06 17:23:38,294 INFO    HandlerThread:2089241 [system_monitor.py:probe():227] Publishing system info
+2025-01-06 17:23:38,294 DEBUG   HandlerThread:2089241 [system_info.py:_save_code():44] Saving code
+2025-01-06 17:23:38,295 DEBUG   HandlerThread:2089241 [system_info.py:_save_code():67] Saving code done
+2025-01-06 17:23:38,295 DEBUG   HandlerThread:2089241 [system_info.py:_save_patches():84] Saving git patches
+2025-01-06 17:23:38,358 DEBUG   HandlerThread:2089241 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-06 17:23:38,358 INFO    HandlerThread:2089241 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-06 17:23:38,361 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:23:38,361 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-06 17:23:38,362 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-06 17:23:38,362 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-06 17:23:38,601 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-06 17:23:38,601 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: python_packages
+2025-01-06 17:23:38,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:23:38,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:23:38,606 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/pvipa42e-diff.patch
+2025-01-06 17:23:38,611 INFO    wandb-upload_0:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/k75qts87-wandb-metadata.json
+2025-01-06 17:23:38,626 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/srh84i2k-code/FISH/eval_robot.py
+2025-01-06 17:23:38,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:38,645 INFO    HandlerThread:2089241 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/tb"
+  save: true
+}
+control {
+  relay_id: "b13bb7481ef74d79a82fde4017db3371"
+}
+_info {
+  stream_id: "9fviy2t8"
+}
+
+2025-01-06 17:23:38,645 DEBUG   HandlerThread:2089241 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-06 17:23:38,703 DEBUG   SenderThread:2089241 [sender.py:send():379] send: telemetry
+2025-01-06 17:23:38,703 DEBUG   SenderThread:2089241 [sender.py:send():379] send: tbrecord
+2025-01-06 17:23:38,704 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:23:38,704 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0 with policy live
+2025-01-06 17:23:38,884 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/yqydwi4g-events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:23:39,178 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-metadata.json
+2025-01-06 17:23:39,178 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/diff.patch
+2025-01-06 17:23:39,178 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:23:39,178 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:23:39,178 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code/FISH/eval_robot.py
+2025-01-06 17:23:39,179 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/requirements.txt
+2025-01-06 17:23:39,179 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code/FISH
+2025-01-06 17:23:39,179 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code
+2025-01-06 17:23:39,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:40,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:41,177 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:23:41,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:42,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:43,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:23:43,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:44,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:45,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:46,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:46,663 DEBUG   SenderThread:2089241 [sender.py:send():379] send: config
+2025-01-06 17:23:47,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:48,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:23:48,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:49,179 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:23:49,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:50,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:51,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:52,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:53,181 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:23:53,601 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:23:53,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:23:53,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:53,726 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:23:54,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:55,181 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:23:55,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:56,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:57,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:58,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:59,578 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:23:59,621 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:00,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:01,183 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:01,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:02,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:03,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:04,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:04,781 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:05,184 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:05,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:06,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:07,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:08,601 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:24:08,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:24:08,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:09,185 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:09,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:10,385 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:10,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:11,186 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml
+2025-01-06 17:24:11,186 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:11,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:12,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:13,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:14,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:15,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:15,981 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:16,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:17,188 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:17,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:18,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:19,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:20,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:21,386 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:21,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:22,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:23,190 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:23,601 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:24:23,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:24:23,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:24,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:25,191 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:25,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:26,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:26,786 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:27,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:28,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:29,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:30,271 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:24:30,271 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4 with policy now
+2025-01-06 17:24:30,271 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:24:30,272 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:24:30,273 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:24:30,274 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:24:30,500 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/nt2usiq9-media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4
+2025-01-06 17:24:30,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:31,193 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:31,193 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:24:31,193 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4
+2025-01-06 17:24:31,194 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos
+2025-01-06 17:24:31,194 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval
+2025-01-06 17:24:31,194 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media
+2025-01-06 17:24:31,457 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/5dym5rfa-media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4
+2025-01-06 17:24:31,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:32,277 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:32,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:33,194 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:33,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:34,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:35,194 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:35,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:36,633 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:37,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:37,634 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:38,278 DEBUG   SystemMonitor:2089241 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-06 17:24:38,280 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:24:38,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:24:38,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:24:38,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:39,196 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:39,610 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:40,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:41,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:42,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:42,697 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:43,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:44,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:45,198 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:24:45,435 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/8b9s1r6g-events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:24:45,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:46,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:47,199 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:47,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:48,028 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:48,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:49,199 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:49,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:50,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:51,200 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:51,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:52,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:53,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:24:53,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:24:53,649 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:53,703 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:54,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:55,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:56,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:57,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:58,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:59,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:59,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:00,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:01,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:02,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:03,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:04,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:05,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:05,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:06,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:07,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:08,292 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:25:08,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:25:08,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:25:08,649 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:09,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:10,615 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:10,740 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:11,615 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:12,616 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:13,616 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:14,616 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:15,617 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:15,818 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:16,373 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:25:16,373 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4 with policy now
+2025-01-06 17:25:16,374 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:25:16,375 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:25:16,375 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:25:16,378 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:25:16,606 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/xp3trwwo-media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4
+2025-01-06 17:25:16,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:17,208 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:17,208 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:25:17,208 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4
+2025-01-06 17:25:17,208 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval
+2025-01-06 17:25:17,488 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/q3b3zwt0-media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4
+2025-01-06 17:25:17,619 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:18,619 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:19,209 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:19,631 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:20,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:21,065 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:21,209 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:21,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:22,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:23,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:25:23,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:25:23,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:24,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:25,211 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:25,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:26,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:26,771 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:27,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:28,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:29,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:30,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:31,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:31,909 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:32,213 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:25:32,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:33,082 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:25:33,082 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:25:33,082 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:25:33,213 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:33,214 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:25:33,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:34,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:35,214 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:35,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:36,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:37,170 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:37,215 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:37,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:38,281 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:25:38,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:25:38,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:25:38,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:39,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:40,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:41,217 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:41,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:42,358 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:42,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:43,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:44,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:45,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:46,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:47,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:47,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:48,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:49,219 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:49,626 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:50,626 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:51,626 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:52,626 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:53,221 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:53,353 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:53,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:25:53,603 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:25:53,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:54,627 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:55,626 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:56,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:57,627 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:58,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:58,627 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:59,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:00,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:01,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:02,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:03,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:03,631 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:04,105 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:26:04,105 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4 with policy now
+2025-01-06 17:26:04,109 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:26:04,109 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:26:04,110 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:26:04,110 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:26:04,224 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:26:04,224 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4
+2025-01-06 17:26:04,225 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval
+2025-01-06 17:26:04,314 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/a3tt0cor-media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4
+2025-01-06 17:26:04,575 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/ak1a8jur-media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4
+2025-01-06 17:26:04,634 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:05,225 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:05,639 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:06,647 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:07,638 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:08,281 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:26:08,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:26:08,606 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:26:08,640 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:08,737 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:09,226 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:09,660 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:10,639 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:11,651 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:12,642 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:13,227 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:13,640 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:14,618 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:14,646 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:15,650 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:16,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:17,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:18,648 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:19,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:20,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:20,647 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:21,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:22,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:23,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:26:23,614 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:26:23,649 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:23,916 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:26:23,917 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:26:23,918 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:26:24,231 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:26:24,232 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:26:24,465 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/clksa26s-events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:26:24,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:25,232 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:25,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:26,565 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:26,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:27,232 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:27,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:28,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:29,233 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:29,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:30,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:31,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:31,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:32,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:33,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:34,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:35,235 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:35,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:36,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:37,236 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:37,361 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:37,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:38,282 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:26:38,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:26:38,605 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:26:38,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:39,237 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:39,666 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:40,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:41,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:42,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:42,665 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:43,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:44,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:45,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:46,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:47,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:48,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:48,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:49,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:50,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:51,656 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:52,656 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:53,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:26:53,604 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:26:53,656 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:53,745 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:54,660 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:55,242 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:55,424 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:26:55,424 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4 with policy now
+2025-01-06 17:26:55,429 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:26:55,430 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:26:55,430 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:26:55,430 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:26:55,637 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/m93omlfj-media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4
+2025-01-06 17:26:55,658 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:56,242 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:26:56,243 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4
+2025-01-06 17:26:56,243 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval
+2025-01-06 17:26:56,503 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/hu8tx640-media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4
+2025-01-06 17:26:56,661 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:57,243 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:57,666 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:58,662 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:59,062 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:59,664 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:00,663 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:01,244 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:01,672 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:02,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:03,245 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:03,681 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:04,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:04,673 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:05,697 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:05,971 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:27:05,971 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:05,972 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:06,246 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:27:06,246 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:27:06,675 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:07,246 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:07,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:08,282 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:27:08,621 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:27:08,621 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:27:08,676 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:09,247 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:09,671 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:09,696 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:10,671 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:11,251 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:11,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:12,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:13,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:14,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:15,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:15,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:16,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:17,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:18,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:19,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:20,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:20,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:21,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:22,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:23,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:27:23,614 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:27:23,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:24,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:25,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:25,732 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:26,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:27,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:28,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:29,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:30,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:31,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:31,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:32,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:33,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:34,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:35,687 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:36,699 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:37,282 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:37,682 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:27:37,683 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4 with policy now
+2025-01-06 17:27:37,687 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:27:37,688 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:27:37,688 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:37,688 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:37,701 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:37,910 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/0ypn3w14-media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4
+2025-01-06 17:27:38,268 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:27:38,268 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4
+2025-01-06 17:27:38,269 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval
+2025-01-06 17:27:38,282 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:27:38,525 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/fsib34ck-media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4
+2025-01-06 17:27:38,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:27:38,622 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:27:38,706 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:39,268 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:39,702 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:40,704 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:41,704 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:42,377 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:42,712 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:43,270 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:43,707 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:44,718 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:45,271 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:45,718 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:46,723 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:47,409 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:47,724 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:48,723 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:49,720 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:50,720 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:51,725 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:52,639 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:52,718 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:53,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:27:53,623 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:27:53,721 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:54,728 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:55,728 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:56,034 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:27:56,034 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:56,034 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:56,274 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:27:56,274 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:27:56,504 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/40pip10a-events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:27:56,751 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:56,991 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 17:27:56,991 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 17:27:57,117 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:27:57,275 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:57,275 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 17:27:57,275 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media
+2025-01-06 17:27:57,275 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table
+2025-01-06 17:27:57,275 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval
+2025-01-06 17:27:57,460 INFO    wandb-upload_2:2089241 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpoqi_p21m
+2025-01-06 17:27:57,470 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/irrij6by-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 17:27:57,495 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 17:27:57,504 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:27:57,720 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:57,848 INFO    SenderThread:2089241 [sender.py:send_request_log_artifact():1525] logged artifact run-9fviy2t8-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkyODYxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMTUyNw==', 'latestArtifact': None}}
+2025-01-06 17:27:57,849 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:27:57,849 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now
+2025-01-06 17:27:57,849 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:57,849 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:27:57,849 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,850 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,850 DEBUG   SenderThread:2089241 [sender.py:send():379] send: telemetry
+2025-01-06 17:27:57,850 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,854 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,854 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,854 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,854 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,855 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,855 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,855 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,855 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,856 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,856 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,856 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,856 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,857 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,857 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,857 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,857 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,857 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,858 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,858 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,858 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,858 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,859 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 17:27:58,094 INFO    wandb-upload_0:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/5pi39ijt-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 17:27:58,197 INFO    wandb-upload_1:2089241 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe21xi7nc
+2025-01-06 17:27:58,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:27:58,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:27:58,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json
+2025-01-06 17:27:58,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval
+2025-01-06 17:27:58,557 INFO    SenderThread:2089241 [sender.py:send_request_log_artifact():1525] logged artifact run-9fviy2t8-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkyODYyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMTUzMA==', 'latestArtifact': None}}
+2025-01-06 17:27:58,557 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:27:58,557 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json with policy now
+2025-01-06 17:27:58,558 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:27:58,558 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:58,558 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:58,558 DEBUG   SenderThread:2089241 [sender.py:send():379] send: exit
+2025-01-06 17:27:58,558 INFO    SenderThread:2089241 [sender.py:send_exit():586] handling exit code: 0
+2025-01-06 17:27:58,558 INFO    SenderThread:2089241 [sender.py:send_exit():588] handling runtime: 260
+2025-01-06 17:27:58,559 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:58,559 INFO    SenderThread:2089241 [sender.py:send_exit():594] send defer
+2025-01-06 17:27:58,559 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:27:58,559 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-06 17:27:58,559 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:27:58,559 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-06 17:27:58,559 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 1
+2025-01-06 17:27:58,559 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:27:58,559 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-06 17:27:58,559 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:27:58,559 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-06 17:27:58,560 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 2
+2025-01-06 17:27:58,560 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:27:58,560 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-06 17:27:58,560 INFO    HandlerThread:2089241 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-06 17:27:58,560 DEBUG   SystemMonitor:2089241 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-06 17:27:58,560 DEBUG   SystemMonitor:2089241 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-06 17:27:58,560 INFO    HandlerThread:2089241 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-06 17:27:58,561 INFO    HandlerThread:2089241 [interfaces.py:finish():200] Joined disk monitor
+2025-01-06 17:27:58,614 INFO    HandlerThread:2089241 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-06 17:27:58,615 INFO    HandlerThread:2089241 [interfaces.py:finish():200] Joined memory monitor
+2025-01-06 17:27:58,615 INFO    HandlerThread:2089241 [interfaces.py:finish():200] Joined network monitor
+2025-01-06 17:27:58,615 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:27:58,615 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-06 17:27:58,615 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 3
+2025-01-06 17:27:58,615 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:27:58,615 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:27:58,615 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-06 17:27:58,616 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:27:58,616 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-06 17:27:58,616 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 4
+2025-01-06 17:27:58,616 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:27:58,616 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-06 17:27:58,745 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/3j7l2yvp-media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json
+2025-01-06 17:27:59,012 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/r3v6j80t-media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json
+2025-01-06 17:27:59,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:59,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:28:01,277 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:28:05,040 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:28:05,041 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:28:05,041 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-06 17:28:05,042 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:05,042 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-06 17:28:05,042 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 5
+2025-01-06 17:28:05,042 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:05,042 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-06 17:28:05,042 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:28:05,043 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:28:05,043 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:28:05,044 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:28:05,045 DEBUG   SenderThread:2089241 [sender.py:send():379] send: summary
+2025-01-06 17:28:05,045 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:28:05,046 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:05,046 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-06 17:28:05,046 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 6
+2025-01-06 17:28:05,046 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:05,046 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-06 17:28:05,046 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:05,046 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-06 17:28:05,049 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:28:05,176 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 7
+2025-01-06 17:28:05,177 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:05,177 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-06 17:28:05,177 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:05,177 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-06 17:28:05,278 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml
+2025-01-06 17:28:05,279 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:28:05,319 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:28:08,561 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 8
+2025-01-06 17:28:08,561 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:28:08,561 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:08,561 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-06 17:28:08,562 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:08,562 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-06 17:28:08,562 INFO    SenderThread:2089241 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-06 17:28:08,564 INFO    SenderThread:2089241 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-06 17:28:08,566 INFO    SenderThread:2089241 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-06 17:28:08,580 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 9
+2025-01-06 17:28:08,580 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:08,580 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-06 17:28:08,580 DEBUG   SenderThread:2089241 [sender.py:send():379] send: artifact
+2025-01-06 17:28:08,767 INFO    SenderThread:2089241 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'versionIndex': 423}}}
+2025-01-06 17:28:08,768 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:08,768 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-06 17:28:08,768 INFO    SenderThread:2089241 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-06 17:28:09,280 INFO    SenderThread:2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:28:09,280 INFO    SenderThread:2089241 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files
+2025-01-06 17:28:09,280 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml config.yaml
+2025-01-06 17:28:09,280 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0 events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:28:09,280 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json wandb-summary.json
+2025-01-06 17:28:09,281 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/requirements.txt requirements.txt
+2025-01-06 17:28:09,282 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-metadata.json wandb-metadata.json
+2025-01-06 17:28:09,282 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log output.log
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/diff.patch diff.patch
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4 media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4 media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4 media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4 media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4 media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4
+2025-01-06 17:28:09,286 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 17:28:09,286 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json
+2025-01-06 17:28:09,286 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 10
+2025-01-06 17:28:09,288 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:09,288 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-06 17:28:09,288 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:09,288 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-06 17:28:09,288 INFO    SenderThread:2089241 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 17:28:09,320 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:28:09,320 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:28:09,455 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml
+2025-01-06 17:28:09,493 INFO    wandb-upload_0:2089241 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:28:09,515 INFO    wandb-upload_3:2089241 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/requirements.txt
+2025-01-06 17:28:09,529 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/1uy4agum-events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:28:09,540 INFO    wandb-upload_4:2089241 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:28:09,740 INFO    Thread-11 (_thread_body):2089241 [sender.py:transition_state():614] send defer: 11
+2025-01-06 17:28:09,740 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:09,741 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-06 17:28:09,741 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:09,741 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-06 17:28:09,741 INFO    SenderThread:2089241 [file_pusher.py:join():175] waiting for file pusher
+2025-01-06 17:28:09,742 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 12
+2025-01-06 17:28:09,742 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:09,742 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-06 17:28:09,742 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:09,742 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-06 17:28:09,742 INFO    SenderThread:2089241 [file_stream.py:finish():601] file stream finish called
+2025-01-06 17:28:09,931 INFO    SenderThread:2089241 [file_stream.py:finish():605] file stream finish is done
+2025-01-06 17:28:09,932 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 13
+2025-01-06 17:28:09,932 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:09,932 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-06 17:28:09,932 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:09,932 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-06 17:28:09,932 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 14
+2025-01-06 17:28:09,932 DEBUG   SenderThread:2089241 [sender.py:send():379] send: final
+2025-01-06 17:28:09,932 DEBUG   SenderThread:2089241 [sender.py:send():379] send: footer
+2025-01-06 17:28:09,932 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:09,933 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-06 17:28:09,933 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:09,933 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-06 17:28:09,933 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:28:09,933 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: server_info
+2025-01-06 17:28:09,934 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:28:09,934 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: server_info
+2025-01-06 17:28:09,937 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:28:09,937 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-06 17:28:09,938 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-06 17:28:09,938 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:28:09,988 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:28:09,989 INFO    MainThread:2089241 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-06 17:28:09,989 INFO    MainThread:2089241 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-06 17:28:09,989 INFO    MainThread:2089241 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-06 17:28:09,989 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-06 17:28:09,990 INFO    HandlerThread:2089241 [handler.py:finish():882] shutting down handler
+2025-01-06 17:28:10,937 INFO    WriterThread:2089241 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/run-9fviy2t8.wandb
+2025-01-06 17:28:10,989 INFO    SenderThread:2089241 [sender.py:finish():1615] shutting down sender
+2025-01-06 17:28:10,989 INFO    SenderThread:2089241 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 17:28:10,989 INFO    SenderThread:2089241 [file_pusher.py:join():175] waiting for file pusher
diff --git a/9fviy2t8/wandb/debug.log b/9fviy2t8/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..b9d93cb197159563ee6bb9f966909f289634d48a
--- /dev/null
+++ b/9fviy2t8/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-06 17:23:37,738 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-06 17:23:37,738 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Configure stats pid to 2089076
+2025-01-06 17:23:37,738 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-06 17:23:37,738 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/settings
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/logs/debug.log
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/logs/debug-internal.log
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:init():569] calling init triggers
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8'}
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:init():619] starting backend
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:init():623] setting up manager
+2025-01-06 17:23:37,743 INFO    MainThread:2089076 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-06 17:23:37,744 INFO    MainThread:2089076 [wandb_init.py:init():631] backend started and connected
+2025-01-06 17:23:37,759 INFO    MainThread:2089076 [wandb_init.py:init():720] updated telemetry
+2025-01-06 17:23:37,765 INFO    MainThread:2089076 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-06 17:23:38,193 INFO    MainThread:2089076 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-06 17:23:38,265 INFO    MainThread:2089076 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-06 17:23:38,265 INFO    MainThread:2089076 [wandb_init.py:init():804] starting run threads in backend
+2025-01-06 17:23:38,602 INFO    MainThread:2089076 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-06 17:23:38,602 INFO    MainThread:2089076 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-06 17:23:38,603 INFO    MainThread:2089076 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-06 17:23:38,603 INFO    MainThread:2089076 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-06 17:23:38,604 INFO    MainThread:2089076 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-06 17:23:38,604 INFO    MainThread:2089076 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/tb, True
+2025-01-06 17:23:46,662 INFO    MainThread:2089076 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-06 17:28:10,990 WARNING MsgRouterThr:2089076 [router.py:message_loop():77] message_loop has been closed
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code/FISH/eval_robot.py b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..45e2d9b6370cc9329c56245c1a1b2000954b9e4f
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 1
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1009_1'
+wandb_notes:
+  desc: null
+  value: 1009_1_restarted_3
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 5
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 1
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1736202217
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: greece
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/diff.patch b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..43dd1a505c5d25a547a09b1d8e6dc31eb2a6a985
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/diff.patch
@@ -0,0 +1,63 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index d2afe29..101f9bd 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint}
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1002_0' 
++wandb_run_id: '1009_1' # seed 1
++# wandb_run_id: '1022_0' # dataset shuffle seed 1
++
++# RGBD+mask+contact(all ftrs)+act history
++# wandb_run_id: '1003_0' 
++# wandb_run_id: '1013_1' # seed 1
++# wandb_run_id: '1017_0' # dataset shuffle seed 1
+ 
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1007_0' 
+-wandb_run_id: '1922_1' # seed 1
++# wandb_run_id: '1922_1' # seed 1
+ # wandb_run_id: '1938_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1
+ # wandb_run_id: '1926_1' # seed 1
+ # wandb_run_id: '1930_0' # dataset shuffle seed 1
+ 
+-
+-
+ # RGBD+mask+contact(contact+DTC)+act history
+ # wandb_run_id: '1745_0'
+ 
++# all books, 4/20 demos per book
++
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+ 
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index c7eadce..e3c7dc1 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -161,8 +161,18 @@ run_id_list = [
+ run_id_list = [
+     # '1922_1',
+     # '1922_2',
+-    '1926_1',
+-    '1930_0',
++    # '1944_0',
++    # '1972_0',
++    # '1976_1',
++    # '1948_0',
++    # '1950_0',
++    # '1954_1',
++    '1002_0',
++    '1009_1',
++    '1022_0',
++    '1003_0',
++    '1013_1',
++    '1017_0',
+ ]
+ 
+ checkpoint_epoch = 12000
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]}
\ No newline at end of file
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..5a592aac7e97094d41fce54046633e127c09be66
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[10, 10, 1.0, 0.9892721175168917, 1, 0.7146403789090983, 1, 0.7492533623673705, 1, 1, 1]]}
\ No newline at end of file
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4 b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..7478195d9e30f7cc4ef1f2e7138fe97ba8a0add6
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fd61fcba1e69d329c7453ca0f5c4e8c027634331e07ca6e286da68c41227226
+size 1142388
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4 b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..a3d0b0c415badc728f2af8bc300ae2fbab93e996
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23c6d820f71940e1e98dc187f36572a14dcf37f4660569737a14ae95fe3fd579
+size 1138983
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4 b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..952819566e460d624dc17d5c35d7c74e0a64c855
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bda5f29eb6d3686af62f4e6a91088d3a306986aaf7ed02443d20bc532e459e3
+size 1151555
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4 b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..9511eeaf42883177445230d2e63aacda8a133212
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f833856d563f00bfb38f835f8817b79977c979df6bc732bc53c7881cd00ff48
+size 1163215
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4 b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..be99ccdd7a9535eb2a96f397988bea42e0357a50
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a47514f65ac4ebdff905da925320694cb5b8cbad4e6f5d8c4bd25194bc310d9b
+size 1140822
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..93205fe1fb4182b263bcc586fdebb8f59506df6d
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
@@ -0,0 +1,273 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1736202226.663460]: resetting environment
+[INFO] [1736202226.666305]: cleared current plan
+[INFO] [1736202226.667145]: moving to home
+[INFO] [1736202227.669209]: reached home
+[INFO] [1736202227.670577]: reset action history
+[INFO] [1736202230.380212]: environment reset
+Starting episode 0
+[INFO] [1736202230.382291]: resetting environment
+[INFO] [1736202230.386166]: cleared current plan
+[INFO] [1736202230.387348]: moving to home
+[INFO] [1736202231.389270]: reached home
+[INFO] [1736202231.390669]: reset action history
+[INFO] [1736202234.097673]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1736202238.567387]: resetting environment
+[INFO] [1736202238.573749]: cleared current plan
+[INFO] [1736202238.575174]: moving to home
+[INFO] [1736202239.578326]: reached home
+[INFO] [1736202239.579632]: reset action history
+[INFO] [1736202242.286150]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/episode_rosbags/episode_0_2025-01-06-17-24-02.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2089076.log
+started roslaunch server http://158.130.50.37:46355/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2089364]
+process[print_text-2]: started with pid [2089365]
+process[rosbag_record-3]: started with pid [2089366]
+started bagging!
+For topic gripper_width: timestamp difference is -108527235 for nearest: 1736202242740148970 - target: 1736202242848676205 at idx 5
+For topic gripper_width: timestamp difference is -107902996 for nearest: 1736202243540129192 - target: 1736202243648032188 at idx 14
+For topic gripper_width: timestamp difference is -114790253 for nearest: 1736202247740157314 - target: 1736202247854947567 at idx 72
+For topic gripper_width: timestamp difference is -111907765 for nearest: 1736202249140157894 - target: 1736202249252065659 at idx 91
+For topic gripper_width: timestamp difference is -200467558 for nearest: 1736202254640131502 - target: 1736202254840599060 at idx 169
+For topic gripper_width: timestamp difference is -101134851 for nearest: 1736202260140094206 - target: 1736202260241229057 at idx 237
+For topic gripper_width: timestamp difference is -100111253 for nearest: 1736202263540129655 - target: 1736202263640240908 at idx 249
+[INFO] [1736202269.321444]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit[pub_text-1] killing on exit
+[INFO] [1736202270.051975]: Stored episode 1.
+[INFO] [1736202270.052526]: Saving video...
+[INFO] [1736202270.257588]: Video saved!
+Starting episode 1
+[INFO] [1736202270.272176]: resetting environment
+[INFO] [1736202270.276144]: cleared current plan
+[INFO] [1736202270.276381]: moving to home
+[INFO] [1736202273.883662]: reached home
+[INFO] [1736202273.884129]: reset action history
+[INFO] [1736202276.590572]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736202284.313870]: resetting environment
+[INFO] [1736202284.316943]: cleared current plan
+[INFO] [1736202284.317141]: moving to home
+[INFO] [1736202285.318680]: reached home
+[INFO] [1736202285.319320]: reset action history
+[INFO] [1736202288.028168]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/episode_rosbags/episode_1_2025-01-06-17-24-48.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2089076.log
+started roslaunch server http://158.130.50.37:40277/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2089476]
+process[print_text-5]: started with pid [2089477]
+process[rosbag_record-6]: started with pid [2089501]
+started bagging!
+[INFO] [1736202315.032884]: Storing episode...
+[rosbag_record-6] killing on exit
+[pub_text-4] killing on exit
+[print_text-5] killing on exit
+[INFO] [1736202315.817789]: Stored episode 2.
+[INFO] [1736202315.818019]: Saving video...
+[WARN] [1736202316.389367]: Plan exhausted
+[INFO] [1736202316.346799]: Video saved!
+Starting episode 2
+[INFO] [1736202316.375904]: resetting environment
+[INFO] [1736202316.400110]: cleared current plan
+[INFO] [1736202316.402722]: moving to home
+[INFO] [1736202320.021227]: reached home
+[INFO] [1736202320.021838]: reset action history
+[INFO] [1736202322.805315]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736202331.928268]: resetting environment
+[INFO] [1736202331.940756]: cleared current plan
+[INFO] [1736202331.942553]: moving to home
+[INFO] [1736202332.945206]: reached home
+[INFO] [1736202332.945664]: reset action history
+[INFO] [1736202335.656015]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/episode_rosbags/episode_2_2025-01-06-17-25-35.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2089076.log
+started roslaunch server http://158.130.50.37:33085/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2089637]
+process[print_text-8]: started with pid [2089638]
+process[rosbag_record-9]: started with pid [2089639]
+started bagging!
+For topic gripper_width: timestamp difference is 102734941 for nearest: 1736202339306828874 - target: 1736202339204093933 at idx 56
+For topic gripper_width: timestamp difference is -103991492 for nearest: 1736202347540080802 - target: 1736202347644072294 at idx 162
+For topic gripper_width: timestamp difference is 102377497 for nearest: 1736202351306824766 - target: 1736202351204447269 at idx 213
+[INFO] [1736202362.875523]: Storing episode...
+[rosbag_record-9] killing on exit
+[pub_text-7] killing on exit[print_text-8] killing on exit
+[INFO] [1736202363.655600]: Stored episode 3.
+[INFO] [1736202363.660801]: Saving video...
+[INFO] [1736202364.072286]: Video saved!
+Starting episode 3
+[INFO] [1736202364.118332]: resetting environment
+[INFO] [1736202364.143070]: cleared current plan
+[INFO] [1736202364.147816]: moving to home
+[INFO] [1736202367.764107]: reached home
+[INFO] [1736202367.770556]: reset action history
+[INFO] [1736202370.552212]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736202383.490464]: resetting environment
+[INFO] [1736202383.513146]: cleared current plan
+[INFO] [1736202383.518305]: moving to home
+[INFO] [1736202384.523213]: reached home
+[INFO] [1736202384.523936]: reset action history
+[INFO] [1736202387.311752]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/episode_rosbags/episode_3_2025-01-06-17-26-27.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2089076.log
+started roslaunch server http://158.130.50.37:34437/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2089751]
+process[print_text-11]: started with pid [2089752]
+process[rosbag_record-12]: started with pid [2089776]
+started bagging!
+For topic gripper_width: timestamp difference is 111285096 for nearest: 1736202393906794911 - target: 1736202393795509815 at idx 96
+For topic gripper_width: timestamp difference is 116058757 for nearest: 1736202396306788852 - target: 1736202396190730095 at idx 125
+For topic gripper_width: timestamp difference is 120123753 for nearest: 1736202398306793341 - target: 1736202398186669588 at idx 151
+[INFO] [1736202414.282041]: Storing episode...
+[rosbag_record-12] killing on exit
+[pub_text-10] killing on exit
+[print_text-11] killing on exit
+[WARN] [1736202415.428667]: Plan exhausted
+[INFO] [1736202415.068207]: Stored episode 4.
+[INFO] [1736202415.071323]: Saving video...
+[INFO] [1736202415.379627]: Video saved!
+Starting episode 4
+[INFO] [1736202415.432661]: resetting environment
+[INFO] [1736202415.456123]: cleared current plan
+[INFO] [1736202415.457725]: moving to home
+[INFO] [1736202419.061311]: reached home
+[INFO] [1736202419.062668]: reset action history
+[INFO] [1736202421.858212]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736202425.691244]: resetting environment
+[INFO] [1736202425.715410]: cleared current plan
+[INFO] [1736202425.719644]: moving to home
+[INFO] [1736202426.726090]: reached home
+[INFO] [1736202426.727257]: reset action history
+[INFO] [1736202429.512215]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/episode_rosbags/episode_4_2025-01-06-17-27-09.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2089076.log
+started roslaunch server http://158.130.50.37:36559/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2089884]
+process[print_text-14]: started with pid [2089885]
+process[rosbag_record-15]: started with pid [2089909]
+started bagging!
+[INFO] [1736202456.512486]: Storing episode...
+[print_text-14] killing on exit
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit
+[INFO] [1736202457.278014]: Stored episode 5.
+[INFO] [1736202457.285030]: Saving video...
+[INFO] [1736202457.626338]: Video saved!
+[INFO] [1736202457.688202]: resetting environment
+[INFO] [1736202457.715112]: cleared current plan
+[INFO] [1736202457.716695]: moving to home
+[INFO] [1736202461.330665]: reached home
+[INFO] [1736202461.355734]: reset action history
+[INFO] [1736202464.363388]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/requirements.txt b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-metadata.json b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..cb153757ca0f62aafc03967fd1a2795c4ab249b8
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-06T22:23:38.288383",
+    "startedAt": "2025-01-06T22:23:37.737194",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3749.00175,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.996,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.983,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4492.017,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.312,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 334.0262336730957
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..c0cfb5c3413573466a253ff3ffada80833494ee3
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "0fd61fcba1e69d329c7453ca0f5c4e8c027634331e07ca6e286da68c41227226", "size": 1142388, "path": "media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4"}, "global_step": 664, "_timestamp": 1736202477.3985913, "_runtime": 259.65411925315857, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "23c6d820f71940e1e98dc187f36572a14dcf37f4660569737a14ae95fe3fd579", "size": 1138983, "path": "media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4"}, "eval/num_success": 5.0, "episode": 10.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "1bda5f29eb6d3686af62f4e6a91088d3a306986aaf7ed02443d20bc532e459e3", "size": 1151555, "path": "media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "4f833856d563f00bfb38f835f8817b79977c979df6bc732bc53c7881cd00ff48", "size": 1163215, "path": "media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "a47514f65ac4ebdff905da925320694cb5b8cbad4e6f5d8c4bd25194bc310d9b", "size": 1140822, "path": "media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://v3054rw5agzd0lggr3okx6pg9qev6d6jdue0wu7ycyu130n37fdq91hbfdtf97uo3w73rs6wm4sgvioz68h6i182p2g5jxkkvc7ipgkvla8onoegx600tc9pvdzis41z/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qf6bjhlxu1a6nwuzxugpv9tdpu6eypsllknw1yj7tqo71updyis6i5d6nzctdt1bllgjwk0wbguewwwjix9sv33jxtj6tqcqu1rsyara6yok6m7ovimp29k6e5h3aaoy:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 10, "total_num_episodes": 10, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9892721175168917, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.7146403789090983, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.7492533623673705, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "802de47402cd97ff16985e7f9caeff0984676b3a683258550eeb2d6f43f4d0e7", "size": 420, "artifact_path": "wandb-client-artifact://wvy6t23qi4bn8hg6ptnjjjinirug5pfejthcud94u908vmj2d3sbh0sy60waclcykm6ecl0s0740i01jp2ifk57trg6b9q8kn2lacam4yrtli7cwkggjjl51cj51ge2g/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://x7byc9j0rmnc8eqot7falr8cvqprnl2o4z31ur5rppr2rjy1rf1grdh36h3a32y9cv75gp2xeo4ld8r132zduhoax702ud831qqpbvwqyth4ouqu9esr054xoprgcnfq:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 260}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.9892721176147461, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.7146403789520264, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.7492533326148987, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0}
\ No newline at end of file
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/logs/debug-internal.log b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..9a119ebb0219ae8cb50396dab79f019389d0ad52
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/logs/debug-internal.log
@@ -0,0 +1,763 @@
+2025-01-06 17:23:37,744 INFO    StreamThr :2089241 [internal.py:wandb_internal():85] W&B internal server running at pid: 2089241, started at: 2025-01-06 17:23:37.744072
+2025-01-06 17:23:37,745 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status
+2025-01-06 17:23:37,760 INFO    WriterThread:2089241 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/run-9fviy2t8.wandb
+2025-01-06 17:23:37,760 DEBUG   SenderThread:2089241 [sender.py:send():379] send: header
+2025-01-06 17:23:37,768 DEBUG   SenderThread:2089241 [sender.py:send():379] send: run
+2025-01-06 17:23:38,176 INFO    SenderThread:2089241 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files
+2025-01-06 17:23:38,176 INFO    SenderThread:2089241 [sender.py:_start_run_threads():1188] run started: 9fviy2t8 with start time 1736202217.744472
+2025-01-06 17:23:38,193 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: check_version
+2025-01-06 17:23:38,194 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: check_version
+2025-01-06 17:23:38,270 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: run_start
+2025-01-06 17:23:38,275 DEBUG   HandlerThread:2089241 [system_info.py:__init__():26] System info init
+2025-01-06 17:23:38,275 DEBUG   HandlerThread:2089241 [system_info.py:__init__():41] System info init done
+2025-01-06 17:23:38,275 INFO    HandlerThread:2089241 [system_monitor.py:start():194] Starting system monitor
+2025-01-06 17:23:38,275 INFO    SystemMonitor:2089241 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-06 17:23:38,275 INFO    SystemMonitor:2089241 [interfaces.py:start():188] Started cpu monitoring
+2025-01-06 17:23:38,276 INFO    HandlerThread:2089241 [system_monitor.py:probe():214] Collecting system info
+2025-01-06 17:23:38,276 INFO    SystemMonitor:2089241 [interfaces.py:start():188] Started disk monitoring
+2025-01-06 17:23:38,277 INFO    SystemMonitor:2089241 [interfaces.py:start():188] Started gpu monitoring
+2025-01-06 17:23:38,277 INFO    SystemMonitor:2089241 [interfaces.py:start():188] Started memory monitoring
+2025-01-06 17:23:38,278 INFO    SystemMonitor:2089241 [interfaces.py:start():188] Started network monitoring
+2025-01-06 17:23:38,288 DEBUG   HandlerThread:2089241 [system_info.py:probe():152] Probing system
+2025-01-06 17:23:38,289 DEBUG   HandlerThread:2089241 [system_info.py:_probe_git():137] Probing git
+2025-01-06 17:23:38,294 DEBUG   HandlerThread:2089241 [system_info.py:_probe_git():145] Probing git done
+2025-01-06 17:23:38,294 DEBUG   HandlerThread:2089241 [system_info.py:probe():200] Probing system done
+2025-01-06 17:23:38,294 DEBUG   HandlerThread:2089241 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:23:38.288383', 'startedAt': '2025-01-06T22:23:37.737194', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.00175, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.996, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.983, 'min': 800.0, 'max': 4700.0}, {'current': 4492.017, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.312, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 334.0262336730957}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-06 17:23:38,294 INFO    HandlerThread:2089241 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-06 17:23:38,294 INFO    HandlerThread:2089241 [system_monitor.py:probe():227] Publishing system info
+2025-01-06 17:23:38,294 DEBUG   HandlerThread:2089241 [system_info.py:_save_code():44] Saving code
+2025-01-06 17:23:38,295 DEBUG   HandlerThread:2089241 [system_info.py:_save_code():67] Saving code done
+2025-01-06 17:23:38,295 DEBUG   HandlerThread:2089241 [system_info.py:_save_patches():84] Saving git patches
+2025-01-06 17:23:38,358 DEBUG   HandlerThread:2089241 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-06 17:23:38,358 INFO    HandlerThread:2089241 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-06 17:23:38,361 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:23:38,361 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-06 17:23:38,362 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-06 17:23:38,362 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-06 17:23:38,601 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-06 17:23:38,601 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: python_packages
+2025-01-06 17:23:38,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:23:38,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:23:38,606 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/pvipa42e-diff.patch
+2025-01-06 17:23:38,611 INFO    wandb-upload_0:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/k75qts87-wandb-metadata.json
+2025-01-06 17:23:38,626 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/srh84i2k-code/FISH/eval_robot.py
+2025-01-06 17:23:38,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:38,645 INFO    HandlerThread:2089241 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/tb"
+  save: true
+}
+control {
+  relay_id: "b13bb7481ef74d79a82fde4017db3371"
+}
+_info {
+  stream_id: "9fviy2t8"
+}
+
+2025-01-06 17:23:38,645 DEBUG   HandlerThread:2089241 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-06 17:23:38,703 DEBUG   SenderThread:2089241 [sender.py:send():379] send: telemetry
+2025-01-06 17:23:38,703 DEBUG   SenderThread:2089241 [sender.py:send():379] send: tbrecord
+2025-01-06 17:23:38,704 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:23:38,704 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0 with policy live
+2025-01-06 17:23:38,884 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/yqydwi4g-events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:23:39,178 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-metadata.json
+2025-01-06 17:23:39,178 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/diff.patch
+2025-01-06 17:23:39,178 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:23:39,178 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:23:39,178 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code/FISH/eval_robot.py
+2025-01-06 17:23:39,179 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/requirements.txt
+2025-01-06 17:23:39,179 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code/FISH
+2025-01-06 17:23:39,179 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code
+2025-01-06 17:23:39,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:40,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:41,177 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:23:41,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:42,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:43,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:23:43,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:44,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:45,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:46,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:46,663 DEBUG   SenderThread:2089241 [sender.py:send():379] send: config
+2025-01-06 17:23:47,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:48,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:23:48,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:49,179 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:23:49,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:50,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:51,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:52,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:53,181 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:23:53,601 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:23:53,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:23:53,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:53,726 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:23:54,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:55,181 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:23:55,603 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:56,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:57,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:58,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:23:59,578 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:23:59,621 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:00,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:01,183 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:01,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:02,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:03,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:04,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:04,781 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:05,184 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:05,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:06,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:07,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:08,601 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:24:08,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:24:08,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:09,185 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:09,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:10,385 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:10,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:11,186 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml
+2025-01-06 17:24:11,186 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:11,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:12,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:13,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:14,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:15,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:15,981 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:16,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:17,188 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:17,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:18,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:19,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:20,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:21,386 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:21,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:22,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:23,190 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:23,601 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:24:23,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:24:23,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:24,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:25,191 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:25,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:26,607 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:26,786 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:27,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:28,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:29,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:30,271 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:24:30,271 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4 with policy now
+2025-01-06 17:24:30,271 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:24:30,272 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:24:30,273 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:24:30,274 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:24:30,500 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/nt2usiq9-media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4
+2025-01-06 17:24:30,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:31,193 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:31,193 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:24:31,193 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4
+2025-01-06 17:24:31,194 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos
+2025-01-06 17:24:31,194 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval
+2025-01-06 17:24:31,194 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media
+2025-01-06 17:24:31,457 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/5dym5rfa-media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4
+2025-01-06 17:24:31,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:32,277 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:32,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:33,194 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:33,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:34,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:35,194 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:35,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:36,633 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:37,609 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:37,634 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:38,278 DEBUG   SystemMonitor:2089241 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-06 17:24:38,280 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:24:38,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:24:38,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:24:38,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:39,196 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:39,610 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:40,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:41,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:42,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:42,697 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:43,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:44,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:45,198 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:24:45,435 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/8b9s1r6g-events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:24:45,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:46,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:47,199 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:47,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:48,028 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:48,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:49,199 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:49,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:50,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:51,200 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:24:51,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:52,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:53,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:24:53,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:24:53,649 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:53,703 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:54,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:55,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:56,612 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:57,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:58,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:24:59,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:24:59,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:00,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:01,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:02,613 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:03,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:04,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:05,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:05,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:06,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:07,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:08,292 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:25:08,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:25:08,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:25:08,649 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:09,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:10,615 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:10,740 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:11,615 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:12,616 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:13,616 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:14,616 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:15,617 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:15,818 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:16,373 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:25:16,373 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4 with policy now
+2025-01-06 17:25:16,374 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:25:16,375 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:25:16,375 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:25:16,378 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:25:16,606 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/xp3trwwo-media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4
+2025-01-06 17:25:16,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:17,208 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:17,208 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:25:17,208 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4
+2025-01-06 17:25:17,208 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval
+2025-01-06 17:25:17,488 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/q3b3zwt0-media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4
+2025-01-06 17:25:17,619 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:18,619 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:19,209 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:19,631 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:20,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:21,065 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:21,209 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:21,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:22,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:23,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:25:23,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:25:23,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:24,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:25,211 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:25,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:26,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:26,771 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:27,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:28,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:29,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:30,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:31,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:31,909 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:32,213 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:25:32,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:33,082 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:25:33,082 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:25:33,082 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:25:33,213 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:33,214 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:25:33,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:34,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:35,214 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:35,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:36,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:37,170 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:37,215 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:37,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:38,281 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:25:38,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:25:38,602 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:25:38,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:39,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:40,624 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:41,217 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:41,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:42,358 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:42,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:43,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:44,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:45,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:46,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:47,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:47,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:48,625 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:49,219 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:49,626 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:50,626 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:51,626 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:52,626 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:53,221 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:25:53,353 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:53,602 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:25:53,603 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:25:53,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:54,627 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:55,626 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:56,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:57,627 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:58,608 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:25:58,627 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:25:59,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:00,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:01,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:02,628 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:03,611 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:03,631 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:04,105 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:26:04,105 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4 with policy now
+2025-01-06 17:26:04,109 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:26:04,109 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:26:04,110 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:26:04,110 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:26:04,224 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:26:04,224 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4
+2025-01-06 17:26:04,225 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval
+2025-01-06 17:26:04,314 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/a3tt0cor-media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4
+2025-01-06 17:26:04,575 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/ak1a8jur-media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4
+2025-01-06 17:26:04,634 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:05,225 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:05,639 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:06,647 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:07,638 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:08,281 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:26:08,606 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:26:08,606 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:26:08,640 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:08,737 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:09,226 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:09,660 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:10,639 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:11,651 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:12,642 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:13,227 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:13,640 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:14,618 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:14,646 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:15,650 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:16,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:17,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:18,648 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:19,645 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:20,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:20,647 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:21,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:22,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:23,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:26:23,614 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:26:23,649 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:23,916 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:26:23,917 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:26:23,918 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:26:24,231 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:26:24,232 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:26:24,465 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/clksa26s-events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:26:24,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:25,232 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:25,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:26,565 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:26,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:27,232 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:27,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:28,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:29,233 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:29,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:30,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:31,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:31,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:32,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:33,652 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:34,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:35,235 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:35,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:36,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:37,236 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:37,361 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:37,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:38,282 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:26:38,605 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:26:38,605 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:26:38,653 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:39,237 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:39,666 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:40,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:41,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:42,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:42,665 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:43,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:44,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:45,654 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:46,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:47,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:48,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:48,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:49,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:50,655 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:51,656 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:52,656 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:53,604 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:26:53,604 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:26:53,656 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:53,745 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:54,660 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:55,242 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:55,424 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:26:55,424 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4 with policy now
+2025-01-06 17:26:55,429 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:26:55,430 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:26:55,430 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:26:55,430 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:26:55,637 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/m93omlfj-media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4
+2025-01-06 17:26:55,658 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:56,242 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:26:56,243 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4
+2025-01-06 17:26:56,243 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval
+2025-01-06 17:26:56,503 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/hu8tx640-media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4
+2025-01-06 17:26:56,661 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:57,243 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:26:57,666 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:58,662 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:26:59,062 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:26:59,664 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:00,663 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:01,244 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:01,672 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:02,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:03,245 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:03,681 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:04,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:04,673 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:05,697 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:05,971 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:27:05,971 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:05,972 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:06,246 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:27:06,246 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:27:06,675 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:07,246 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:07,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:08,282 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:27:08,621 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:27:08,621 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:27:08,676 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:09,247 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:09,671 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:09,696 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:10,671 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:11,251 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:11,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:12,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:13,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:14,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:15,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:15,677 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:16,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:17,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:18,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:19,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:20,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:20,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:21,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:22,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:23,614 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:27:23,614 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:27:23,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:24,678 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:25,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:25,732 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:26,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:27,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:28,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:29,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:30,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:31,623 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:31,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:32,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:33,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:34,679 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:35,687 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:36,699 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:37,282 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:37,682 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:27:37,683 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4 with policy now
+2025-01-06 17:27:37,687 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:27:37,688 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:27:37,688 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:37,688 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:37,701 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:37,910 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/0ypn3w14-media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4
+2025-01-06 17:27:38,268 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:27:38,268 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4
+2025-01-06 17:27:38,269 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval
+2025-01-06 17:27:38,282 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:27:38,525 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/fsib34ck-media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4
+2025-01-06 17:27:38,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:27:38,622 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:27:38,706 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:39,268 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:39,702 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:40,704 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:41,704 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:42,377 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:42,712 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:43,270 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:43,707 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:44,718 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:45,271 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:45,718 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:46,723 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:47,409 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:47,724 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:48,723 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:49,720 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:50,720 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:51,725 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:52,639 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:52,718 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:53,622 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:27:53,623 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:27:53,721 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:54,728 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:55,728 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:56,034 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:27:56,034 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:56,034 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:56,274 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:27:56,274 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:27:56,504 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/40pip10a-events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:27:56,751 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:56,991 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 17:27:56,991 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 17:27:57,117 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:27:57,275 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:57,275 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 17:27:57,275 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media
+2025-01-06 17:27:57,275 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table
+2025-01-06 17:27:57,275 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval
+2025-01-06 17:27:57,460 INFO    wandb-upload_2:2089241 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpoqi_p21m
+2025-01-06 17:27:57,470 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/irrij6by-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 17:27:57,495 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 17:27:57,504 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:27:57,720 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:27:57,848 INFO    SenderThread:2089241 [sender.py:send_request_log_artifact():1525] logged artifact run-9fviy2t8-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkyODYxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMTUyNw==', 'latestArtifact': None}}
+2025-01-06 17:27:57,849 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:27:57,849 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now
+2025-01-06 17:27:57,849 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:27:57,849 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:27:57,849 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,850 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,850 DEBUG   SenderThread:2089241 [sender.py:send():379] send: telemetry
+2025-01-06 17:27:57,850 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,854 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,854 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,854 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,854 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,855 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,855 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,855 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,855 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,856 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,856 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,856 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,856 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,857 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,857 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,857 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,857 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,857 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,858 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,858 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,858 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:57,858 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:57,859 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 17:27:58,094 INFO    wandb-upload_0:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/5pi39ijt-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 17:27:58,197 INFO    wandb-upload_1:2089241 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe21xi7nc
+2025-01-06 17:27:58,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:27:58,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:27:58,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json
+2025-01-06 17:27:58,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval
+2025-01-06 17:27:58,557 INFO    SenderThread:2089241 [sender.py:send_request_log_artifact():1525] logged artifact run-9fviy2t8-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkyODYyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMTUzMA==', 'latestArtifact': None}}
+2025-01-06 17:27:58,557 DEBUG   SenderThread:2089241 [sender.py:send():379] send: files
+2025-01-06 17:27:58,557 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json with policy now
+2025-01-06 17:27:58,558 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:27:58,558 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:27:58,558 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:58,558 DEBUG   SenderThread:2089241 [sender.py:send():379] send: exit
+2025-01-06 17:27:58,558 INFO    SenderThread:2089241 [sender.py:send_exit():586] handling exit code: 0
+2025-01-06 17:27:58,558 INFO    SenderThread:2089241 [sender.py:send_exit():588] handling runtime: 260
+2025-01-06 17:27:58,559 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:27:58,559 INFO    SenderThread:2089241 [sender.py:send_exit():594] send defer
+2025-01-06 17:27:58,559 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:27:58,559 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-06 17:27:58,559 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:27:58,559 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-06 17:27:58,559 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 1
+2025-01-06 17:27:58,559 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:27:58,559 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-06 17:27:58,559 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:27:58,559 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-06 17:27:58,560 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 2
+2025-01-06 17:27:58,560 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:27:58,560 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-06 17:27:58,560 INFO    HandlerThread:2089241 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-06 17:27:58,560 DEBUG   SystemMonitor:2089241 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-06 17:27:58,560 DEBUG   SystemMonitor:2089241 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-06 17:27:58,560 INFO    HandlerThread:2089241 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-06 17:27:58,561 INFO    HandlerThread:2089241 [interfaces.py:finish():200] Joined disk monitor
+2025-01-06 17:27:58,614 INFO    HandlerThread:2089241 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-06 17:27:58,615 INFO    HandlerThread:2089241 [interfaces.py:finish():200] Joined memory monitor
+2025-01-06 17:27:58,615 INFO    HandlerThread:2089241 [interfaces.py:finish():200] Joined network monitor
+2025-01-06 17:27:58,615 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:27:58,615 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-06 17:27:58,615 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 3
+2025-01-06 17:27:58,615 DEBUG   SenderThread:2089241 [sender.py:send():379] send: stats
+2025-01-06 17:27:58,615 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:27:58,615 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-06 17:27:58,616 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:27:58,616 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-06 17:27:58,616 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 4
+2025-01-06 17:27:58,616 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:27:58,616 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-06 17:27:58,745 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/3j7l2yvp-media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json
+2025-01-06 17:27:59,012 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/r3v6j80t-media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json
+2025-01-06 17:27:59,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:27:59,276 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:28:01,277 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:28:05,040 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:28:05,041 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:28:05,041 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-06 17:28:05,042 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:05,042 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-06 17:28:05,042 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 5
+2025-01-06 17:28:05,042 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:05,042 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-06 17:28:05,042 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:28:05,043 DEBUG   SenderThread:2089241 [sender.py:send():379] send: history
+2025-01-06 17:28:05,043 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:28:05,044 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:28:05,045 DEBUG   SenderThread:2089241 [sender.py:send():379] send: summary
+2025-01-06 17:28:05,045 INFO    SenderThread:2089241 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:28:05,046 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:05,046 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-06 17:28:05,046 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 6
+2025-01-06 17:28:05,046 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:05,046 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-06 17:28:05,046 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:05,046 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-06 17:28:05,049 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:28:05,176 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 7
+2025-01-06 17:28:05,177 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:05,177 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-06 17:28:05,177 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:05,177 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-06 17:28:05,278 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml
+2025-01-06 17:28:05,279 INFO    Thread-12 :2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:28:05,319 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:28:08,561 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 8
+2025-01-06 17:28:08,561 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:28:08,561 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:08,561 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-06 17:28:08,562 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:08,562 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-06 17:28:08,562 INFO    SenderThread:2089241 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-06 17:28:08,564 INFO    SenderThread:2089241 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-06 17:28:08,566 INFO    SenderThread:2089241 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-06 17:28:08,580 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 9
+2025-01-06 17:28:08,580 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:08,580 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-06 17:28:08,580 DEBUG   SenderThread:2089241 [sender.py:send():379] send: artifact
+2025-01-06 17:28:08,767 INFO    SenderThread:2089241 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'versionIndex': 423}}}
+2025-01-06 17:28:08,768 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:08,768 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-06 17:28:08,768 INFO    SenderThread:2089241 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-06 17:28:09,280 INFO    SenderThread:2089241 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:28:09,280 INFO    SenderThread:2089241 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files
+2025-01-06 17:28:09,280 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml config.yaml
+2025-01-06 17:28:09,280 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0 events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:28:09,280 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json wandb-summary.json
+2025-01-06 17:28:09,281 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/requirements.txt requirements.txt
+2025-01-06 17:28:09,282 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-metadata.json wandb-metadata.json
+2025-01-06 17:28:09,282 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log output.log
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/diff.patch diff.patch
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4 media/videos/eval/2_eval_2_1bda5f29eb6d3686af62.mp4
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4 media/videos/eval/1_eval_1_23c6d820f71940e1e98d.mp4
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4 media/videos/eval/4_eval_4_a47514f65ac4ebdff905.mp4
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4 media/videos/eval/3_eval_3_4f833856d563f00bfb38.mp4
+2025-01-06 17:28:09,283 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4 media/videos/eval/0_eval_0_0fd61fcba1e69d329c74.mp4
+2025-01-06 17:28:09,286 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 17:28:09,286 INFO    SenderThread:2089241 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json media/table/eval/total_success_rate_ci_6_802de47402cd97ff1698.table.json
+2025-01-06 17:28:09,286 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 10
+2025-01-06 17:28:09,288 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:09,288 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-06 17:28:09,288 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:09,288 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-06 17:28:09,288 INFO    SenderThread:2089241 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 17:28:09,320 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:28:09,320 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:28:09,455 INFO    wandb-upload_2:2089241 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/config.yaml
+2025-01-06 17:28:09,493 INFO    wandb-upload_0:2089241 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/wandb-summary.json
+2025-01-06 17:28:09,515 INFO    wandb-upload_3:2089241 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/requirements.txt
+2025-01-06 17:28:09,529 INFO    wandb-upload_1:2089241 [upload_job.py:push():130] Uploaded file /tmp/tmplqupg89owandb/1uy4agum-events.out.tfevents.1736202218.leonmkim-ROG-Strix-G15CS-G15CS.2089076.0
+2025-01-06 17:28:09,540 INFO    wandb-upload_4:2089241 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/files/output.log
+2025-01-06 17:28:09,740 INFO    Thread-11 (_thread_body):2089241 [sender.py:transition_state():614] send defer: 11
+2025-01-06 17:28:09,740 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:09,741 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-06 17:28:09,741 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:09,741 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-06 17:28:09,741 INFO    SenderThread:2089241 [file_pusher.py:join():175] waiting for file pusher
+2025-01-06 17:28:09,742 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 12
+2025-01-06 17:28:09,742 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:09,742 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-06 17:28:09,742 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:09,742 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-06 17:28:09,742 INFO    SenderThread:2089241 [file_stream.py:finish():601] file stream finish called
+2025-01-06 17:28:09,931 INFO    SenderThread:2089241 [file_stream.py:finish():605] file stream finish is done
+2025-01-06 17:28:09,932 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 13
+2025-01-06 17:28:09,932 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:09,932 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-06 17:28:09,932 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:09,932 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-06 17:28:09,932 INFO    SenderThread:2089241 [sender.py:transition_state():614] send defer: 14
+2025-01-06 17:28:09,932 DEBUG   SenderThread:2089241 [sender.py:send():379] send: final
+2025-01-06 17:28:09,932 DEBUG   SenderThread:2089241 [sender.py:send():379] send: footer
+2025-01-06 17:28:09,932 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:28:09,933 INFO    HandlerThread:2089241 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-06 17:28:09,933 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:28:09,933 INFO    SenderThread:2089241 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-06 17:28:09,933 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:28:09,933 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: server_info
+2025-01-06 17:28:09,934 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:28:09,934 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: server_info
+2025-01-06 17:28:09,937 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:28:09,937 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-06 17:28:09,938 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-06 17:28:09,938 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:28:09,988 DEBUG   SenderThread:2089241 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:28:09,989 INFO    MainThread:2089241 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-06 17:28:09,989 INFO    MainThread:2089241 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-06 17:28:09,989 INFO    MainThread:2089241 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-06 17:28:09,989 DEBUG   HandlerThread:2089241 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-06 17:28:09,990 INFO    HandlerThread:2089241 [handler.py:finish():882] shutting down handler
+2025-01-06 17:28:10,937 INFO    WriterThread:2089241 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/run-9fviy2t8.wandb
+2025-01-06 17:28:10,989 INFO    SenderThread:2089241 [sender.py:finish():1615] shutting down sender
+2025-01-06 17:28:10,989 INFO    SenderThread:2089241 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 17:28:10,989 INFO    SenderThread:2089241 [file_pusher.py:join():175] waiting for file pusher
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/logs/debug.log b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..b9d93cb197159563ee6bb9f966909f289634d48a
--- /dev/null
+++ b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-06 17:23:37,738 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-06 17:23:37,738 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Configure stats pid to 2089076
+2025-01-06 17:23:37,738 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-06 17:23:37,738 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/settings
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/logs/debug.log
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/logs/debug-internal.log
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:init():569] calling init triggers
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8'}
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:init():619] starting backend
+2025-01-06 17:23:37,739 INFO    MainThread:2089076 [wandb_init.py:init():623] setting up manager
+2025-01-06 17:23:37,743 INFO    MainThread:2089076 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-06 17:23:37,744 INFO    MainThread:2089076 [wandb_init.py:init():631] backend started and connected
+2025-01-06 17:23:37,759 INFO    MainThread:2089076 [wandb_init.py:init():720] updated telemetry
+2025-01-06 17:23:37,765 INFO    MainThread:2089076 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-06 17:23:38,193 INFO    MainThread:2089076 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-06 17:23:38,265 INFO    MainThread:2089076 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-06 17:23:38,265 INFO    MainThread:2089076 [wandb_init.py:init():804] starting run threads in backend
+2025-01-06 17:23:38,602 INFO    MainThread:2089076 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-06 17:23:38,602 INFO    MainThread:2089076 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-06 17:23:38,603 INFO    MainThread:2089076 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-06 17:23:38,603 INFO    MainThread:2089076 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-06 17:23:38,604 INFO    MainThread:2089076 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-06 17:23:38,604 INFO    MainThread:2089076 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9fviy2t8/tb, True
+2025-01-06 17:23:46,662 INFO    MainThread:2089076 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-06 17:28:10,990 WARNING MsgRouterThr:2089076 [router.py:message_loop():77] message_loop has been closed
diff --git a/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/run-9fviy2t8.wandb b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/run-9fviy2t8.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..c9c7b34df203fcbebb11f014e165c55b2a095845
Binary files /dev/null and b/9fviy2t8/wandb/run-20250106_172337-9fviy2t8/run-9fviy2t8.wandb differ
diff --git a/9ig9megf/.hydra/config.yaml b/9ig9megf/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab
--- /dev/null
+++ b/9ig9megf/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1009_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/9ig9megf/.hydra/hydra.yaml b/9ig9megf/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..aa8708206fe869684f82ffee4d93ab07274bf5b9
--- /dev/null
+++ b/9ig9megf/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/9ig9megf/.hydra/overrides.yaml b/9ig9megf/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/9ig9megf/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/9ig9megf/episode_rosbags/aligned_depth_to_color_K.npy b/9ig9megf/episode_rosbags/aligned_depth_to_color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/9ig9megf/episode_rosbags/aligned_depth_to_color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/9ig9megf/episode_rosbags/cam_tf_world.npy b/9ig9megf/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/9ig9megf/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/9ig9megf/episode_rosbags/color_K.npy b/9ig9megf/episode_rosbags/color_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604
--- /dev/null
+++ b/9ig9megf/episode_rosbags/color_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200
diff --git a/9ig9megf/episode_rosbags/depth_K.npy b/9ig9megf/episode_rosbags/depth_K.npy
new file mode 100644
index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce
--- /dev/null
+++ b/9ig9megf/episode_rosbags/depth_K.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200
diff --git a/9ig9megf/eval_robot.log b/9ig9megf/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..12d9bef80e5bf903372df5294e80b363b4956376
--- /dev/null
+++ b/9ig9megf/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-06 22:13:16,559][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-06 22:13:16,562][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/9ig9megf/eval_video/0_eval.mp4 b/9ig9megf/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1c63efc59d438bb63dae93fee450bf240520ecea
--- /dev/null
+++ b/9ig9megf/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44f7c849dd38a4c00ad495c118149a4e0b9bc9f0dcde5b0b2df02a50345eefef
+size 1128228
diff --git a/9ig9megf/eval_video/1_eval.mp4 b/9ig9megf/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..d0233c45b00a3bace8cc4b1a5d9576eb82eab306
--- /dev/null
+++ b/9ig9megf/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c32d361c2cac363e3ab2b1a5610d022314c98381ef4cc7e8ac72037370502f6
+size 1134779
diff --git a/9ig9megf/eval_video/2_eval.mp4 b/9ig9megf/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4ef69ab9875fba0d84665befd8d2b2ae1e8c094b
--- /dev/null
+++ b/9ig9megf/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4214c17fed8f2dd3c5b4f1652933de7dcf6275b143d806648990f82e86f77e2f
+size 1135935
diff --git a/9ig9megf/eval_video/3_eval.mp4 b/9ig9megf/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1677e899487ad0cf6212b5154dff27a5c2fbcbf5
--- /dev/null
+++ b/9ig9megf/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a87ee96aee6886a00c6c376db2338c44627415aa14577c6e00666d529ba4bbc
+size 1174830
diff --git a/9ig9megf/eval_video/4_eval.mp4 b/9ig9megf/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f166054f207a9a2aa09dc143b533535eb110681d
--- /dev/null
+++ b/9ig9megf/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eed30eb0a971980a2bf81177202b399340800a8766d47fdba2febd8956f5f421
+size 1144795
diff --git a/9ig9megf/tb/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0 b/9ig9megf/tb/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
new file mode 100644
index 0000000000000000000000000000000000000000..e236390091f77bbccc3c333347480b34eec436a9
--- /dev/null
+++ b/9ig9megf/tb/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:463ac2041b1e7abb3bfd71065b393f004b377a32432e16356f52b2b60000b71a
+size 6539
diff --git a/9ig9megf/wandb/debug-internal.log b/9ig9megf/wandb/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..c7988606b7e7669ab05c76277040cea71b90eda0
--- /dev/null
+++ b/9ig9megf/wandb/debug-internal.log
@@ -0,0 +1,755 @@
+2025-01-06 22:13:22,748 INFO    StreamThr :2122155 [internal.py:wandb_internal():85] W&B internal server running at pid: 2122155, started at: 2025-01-06 22:13:22.748336
+2025-01-06 22:13:22,750 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status
+2025-01-06 22:13:22,763 INFO    WriterThread:2122155 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/run-9ig9megf.wandb
+2025-01-06 22:13:22,763 DEBUG   SenderThread:2122155 [sender.py:send():379] send: header
+2025-01-06 22:13:22,771 DEBUG   SenderThread:2122155 [sender.py:send():379] send: run
+2025-01-06 22:13:23,035 INFO    SenderThread:2122155 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files
+2025-01-06 22:13:23,035 INFO    SenderThread:2122155 [sender.py:_start_run_threads():1188] run started: 9ig9megf with start time 1736219602.749401
+2025-01-06 22:13:23,054 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: check_version
+2025-01-06 22:13:23,054 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: check_version
+2025-01-06 22:13:23,105 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: run_start
+2025-01-06 22:13:23,110 DEBUG   HandlerThread:2122155 [system_info.py:__init__():26] System info init
+2025-01-06 22:13:23,110 DEBUG   HandlerThread:2122155 [system_info.py:__init__():41] System info init done
+2025-01-06 22:13:23,111 INFO    HandlerThread:2122155 [system_monitor.py:start():194] Starting system monitor
+2025-01-06 22:13:23,111 INFO    SystemMonitor:2122155 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-06 22:13:23,111 INFO    HandlerThread:2122155 [system_monitor.py:probe():214] Collecting system info
+2025-01-06 22:13:23,111 INFO    SystemMonitor:2122155 [interfaces.py:start():188] Started cpu monitoring
+2025-01-06 22:13:23,112 INFO    SystemMonitor:2122155 [interfaces.py:start():188] Started disk monitoring
+2025-01-06 22:13:23,112 INFO    SystemMonitor:2122155 [interfaces.py:start():188] Started gpu monitoring
+2025-01-06 22:13:23,113 INFO    SystemMonitor:2122155 [interfaces.py:start():188] Started memory monitoring
+2025-01-06 22:13:23,113 INFO    SystemMonitor:2122155 [interfaces.py:start():188] Started network monitoring
+2025-01-06 22:13:23,124 DEBUG   HandlerThread:2122155 [system_info.py:probe():152] Probing system
+2025-01-06 22:13:23,125 DEBUG   HandlerThread:2122155 [system_info.py:_probe_git():137] Probing git
+2025-01-06 22:13:23,130 DEBUG   HandlerThread:2122155 [system_info.py:_probe_git():145] Probing git done
+2025-01-06 22:13:23,130 DEBUG   HandlerThread:2122155 [system_info.py:probe():200] Probing system done
+2025-01-06 22:13:23,130 DEBUG   HandlerThread:2122155 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:13:23.124287', 'startedAt': '2025-01-07T03:13:22.738878', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.5340000000006, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4491.899, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.29, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.995, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 558.2445449829102}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-06 22:13:23,130 INFO    HandlerThread:2122155 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-06 22:13:23,130 INFO    HandlerThread:2122155 [system_monitor.py:probe():227] Publishing system info
+2025-01-06 22:13:23,130 DEBUG   HandlerThread:2122155 [system_info.py:_save_code():44] Saving code
+2025-01-06 22:13:23,131 DEBUG   HandlerThread:2122155 [system_info.py:_save_code():67] Saving code done
+2025-01-06 22:13:23,132 DEBUG   HandlerThread:2122155 [system_info.py:_save_patches():84] Saving git patches
+2025-01-06 22:13:23,192 DEBUG   HandlerThread:2122155 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-06 22:13:23,193 INFO    HandlerThread:2122155 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-06 22:13:23,196 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:13:23,196 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-06 22:13:23,196 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-06 22:13:23,197 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-06 22:13:23,440 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/812ddovu-wandb-metadata.json
+2025-01-06 22:13:23,464 INFO    wandb-upload_2:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/8bsaca9a-diff.patch
+2025-01-06 22:13:23,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-06 22:13:23,467 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: python_packages
+2025-01-06 22:13:23,468 DEBUG   SenderThread:2122155 [sender.py:send():379] send: telemetry
+2025-01-06 22:13:23,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:13:23,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:13:23,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:23,470 INFO    HandlerThread:2122155 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/tb"
+  save: true
+}
+control {
+  relay_id: "d08939d596444ee6b2c634f165868670"
+}
+_info {
+  stream_id: "9ig9megf"
+}
+
+2025-01-06 22:13:23,470 DEBUG   HandlerThread:2122155 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-06 22:13:23,488 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/oh5ciig3-code/FISH/eval_robot.py
+2025-01-06 22:13:23,570 DEBUG   SenderThread:2122155 [sender.py:send():379] send: tbrecord
+2025-01-06 22:13:23,571 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:13:23,571 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0 with policy live
+2025-01-06 22:13:23,758 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/ig0mgqma-events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:13:24,036 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code/FISH/eval_robot.py
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/requirements.txt
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-metadata.json
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/diff.patch
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code/FISH
+2025-01-06 22:13:24,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:25,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:26,036 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:26,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:27,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:28,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:28,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:29,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:30,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:31,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:32,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:33,334 DEBUG   SenderThread:2122155 [sender.py:send():379] send: config
+2025-01-06 22:13:33,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:33,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:34,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:35,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:36,039 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:36,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:37,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:38,040 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:38,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:13:38,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:13:38,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:38,535 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:39,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:40,040 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:40,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:41,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:42,041 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:42,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:43,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:44,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:44,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:45,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:46,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:47,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:48,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:49,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:49,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:50,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:51,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:52,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:53,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:13:53,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:13:53,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:54,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:54,563 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:55,044 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml
+2025-01-06 22:13:55,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:56,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:57,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:58,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:59,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:00,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:00,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:01,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:02,046 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:02,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:03,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:04,047 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:04,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:05,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:05,888 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:06,048 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:06,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:07,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:08,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:14:08,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:14:08,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:09,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:10,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:11,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:11,581 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:12,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:13,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:14,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:15,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:16,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:17,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:17,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:18,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:19,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:20,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:21,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:22,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:23,114 DEBUG   SystemMonitor:2122155 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-06 22:14:23,114 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:14:23,115 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:23,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:14:23,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:14:23,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:24,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:25,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:26,476 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:27,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:28,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:28,476 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:29,476 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:30,476 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:31,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:32,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:32,601 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:14:32,602 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:14:32,602 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4 with policy now
+2025-01-06 22:14:32,602 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:14:32,602 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:14:32,603 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:14:32,859 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/lil8d117-media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4
+2025-01-06 22:14:33,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4
+2025-01-06 22:14:33,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:14:33,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media
+2025-01-06 22:14:33,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval
+2025-01-06 22:14:33,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos
+2025-01-06 22:14:33,331 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/41hp5crh-media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4
+2025-01-06 22:14:33,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:33,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:34,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:34,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:35,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:36,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:37,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:38,056 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:38,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:14:38,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:14:38,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:38,539 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:39,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:40,057 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:40,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:41,478 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:42,478 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:43,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:44,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:44,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:45,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:46,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:47,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:48,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:49,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:50,061 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:50,061 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:14:50,087 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:50,304 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/85bojlbt-events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:14:50,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:51,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:52,061 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:52,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:53,115 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:14:53,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:14:53,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:14:53,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:54,062 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:54,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:55,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:55,527 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:56,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:57,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:58,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:59,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:00,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:00,768 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:01,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:02,065 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:02,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:03,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:04,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:05,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:06,413 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:06,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:07,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:08,067 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:08,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:15:08,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:15:08,513 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:09,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:10,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:11,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:11,593 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:12,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:13,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:14,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:15,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:16,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:17,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:17,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:18,483 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:19,483 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:20,483 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:21,174 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:15:21,174 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4 with policy now
+2025-01-06 22:15:21,178 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:15:21,179 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:15:21,179 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:15:21,179 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:15:21,426 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/ty8szote-media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4
+2025-01-06 22:15:21,485 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:22,071 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:22,071 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:15:22,071 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4
+2025-01-06 22:15:22,072 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval
+2025-01-06 22:15:22,426 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/9vjyf2vq-media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4
+2025-01-06 22:15:22,484 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:23,115 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:15:23,115 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:23,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:15:23,469 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:15:23,513 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:24,072 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:24,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:25,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:26,072 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:26,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:27,486 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:28,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:28,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:29,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:30,074 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:30,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:31,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:31,932 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:15:31,932 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:15:31,932 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:15:32,075 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:15:32,075 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:15:32,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:33,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:33,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:34,075 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:34,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:35,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:36,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:37,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:38,077 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:38,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:15:38,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:15:38,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:38,535 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:39,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:40,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:41,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:42,078 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:42,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:43,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:44,079 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:44,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:44,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:45,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:46,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:47,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:48,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:49,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:49,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:50,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:51,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:52,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:53,116 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:15:53,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:15:53,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:15:53,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:54,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:54,601 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:55,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:56,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:57,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:58,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:59,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:00,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:00,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:01,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:02,491 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:03,375 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:16:03,375 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4 with policy now
+2025-01-06 22:16:03,377 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:16:03,378 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:16:03,378 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:16:03,379 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:16:03,491 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:03,671 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/2m9ytad9-media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4
+2025-01-06 22:16:04,087 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:04,087 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:16:04,087 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4
+2025-01-06 22:16:04,087 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval
+2025-01-06 22:16:04,385 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/p42gscn4-media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4
+2025-01-06 22:16:04,492 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:05,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:05,497 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:06,087 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:06,500 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:07,501 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:08,088 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:08,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:16:08,471 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:16:08,513 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:09,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:10,505 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:10,781 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:11,506 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:11,757 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:16:11,757 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:16:11,775 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:16:12,090 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:12,090 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:16:12,090 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:16:12,329 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/832f55tw-events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:16:12,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:13,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:14,090 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:14,512 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:15,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:16,003 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:16,091 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:16,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:17,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:18,092 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:18,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:19,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:20,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:21,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:21,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:22,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:23,116 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:16:23,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:16:23,470 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:16:23,513 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:24,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:25,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:26,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:26,599 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:27,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:28,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:29,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:30,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:31,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:32,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:32,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:33,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:34,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:35,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:36,098 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:36,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:37,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:38,229 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:38,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:16:38,470 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:16:38,517 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:39,510 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:40,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:41,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:42,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:43,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:43,516 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:44,057 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:16:44,057 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4 with policy now
+2025-01-06 22:16:44,061 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:16:44,061 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:16:44,061 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:16:44,062 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:16:44,100 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:44,101 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:16:44,101 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4
+2025-01-06 22:16:44,101 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval
+2025-01-06 22:16:44,312 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/6xeimi59-media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4
+2025-01-06 22:16:44,514 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:44,592 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/3xd02x84-media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4
+2025-01-06 22:16:45,517 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:46,101 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:46,514 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:47,518 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:48,519 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:49,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:49,523 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:50,102 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:50,519 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:51,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:52,103 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:52,520 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:53,117 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:16:53,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:16:53,473 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:16:53,523 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:54,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:54,539 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:55,523 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:56,104 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:16:56,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:56,813 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:16:56,813 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:16:56,813 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:16:57,105 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:16:57,526 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:58,105 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:58,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:59,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:59,762 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:00,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:01,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:02,107 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:02,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:03,528 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:04,528 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:05,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:05,528 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:06,528 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:07,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:08,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:17:08,470 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:17:08,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:09,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:10,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:10,536 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:11,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:12,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:13,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:14,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:15,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:16,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:16,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:17,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:18,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:19,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:20,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:21,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:21,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:22,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:23,117 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:17:23,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:17:23,470 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:17:23,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:24,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:25,532 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:26,532 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:26,539 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:27,536 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:28,118 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:28,333 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:17:28,333 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4 with policy now
+2025-01-06 22:17:28,337 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:17:28,338 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:17:28,338 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:28,338 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:28,532 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:28,585 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/17fgcz54-media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4
+2025-01-06 22:17:29,119 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:17:29,119 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4
+2025-01-06 22:17:29,119 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval
+2025-01-06 22:17:29,409 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/uh1fc3so-media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4
+2025-01-06 22:17:29,534 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:30,119 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:30,537 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:31,542 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:31,963 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:32,544 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:33,551 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:34,120 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:34,548 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:35,546 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:36,121 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:36,552 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:37,320 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:37,550 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:37,775 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 22:17:37,775 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 22:17:37,947 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:17:38,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:38,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:17:38,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 22:17:38,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media
+2025-01-06 22:17:38,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table
+2025-01-06 22:17:38,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval
+2025-01-06 22:17:38,336 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/k02egdlf-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 22:17:38,346 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 22:17:38,355 INFO    wandb-upload_2:2122155 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplup4j1go
+2025-01-06 22:17:38,358 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:17:38,360 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/gylawia1-events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:17:38,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:17:38,547 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:38,744 INFO    SenderThread:2122155 [sender.py:send_request_log_artifact():1525] logged artifact run-9ig9megf-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU2MDY2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDczNTY1Nw==', 'latestArtifact': None}}
+2025-01-06 22:17:38,744 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:17:38,744 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now
+2025-01-06 22:17:38,745 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:17:38,745 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,745 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,746 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:17:38,746 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,749 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,749 DEBUG   SenderThread:2122155 [sender.py:send():379] send: telemetry
+2025-01-06 22:17:38,749 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,750 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,750 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,750 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,750 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,751 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,751 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,751 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,751 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,752 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,752 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,752 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,752 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,753 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,753 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,753 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,753 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,754 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,754 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,754 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,754 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,755 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,755 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 22:17:38,991 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/oa86qmqk-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 22:17:39,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:17:39,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:17:39,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json
+2025-01-06 22:17:39,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval
+2025-01-06 22:17:39,132 INFO    wandb-upload_1:2122155 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpeqtnnbg3
+2025-01-06 22:17:39,306 INFO    wandb-upload_2:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/m76450mq-media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json
+2025-01-06 22:17:39,535 INFO    SenderThread:2122155 [sender.py:send_request_log_artifact():1525] logged artifact run-9ig9megf-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU2MDY4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDczNTY2MA==', 'latestArtifact': None}}
+2025-01-06 22:17:39,535 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:17:39,536 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json with policy now
+2025-01-06 22:17:39,536 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:17:39,536 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:39,537 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:39,537 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:17:39,547 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:39,741 DEBUG   SenderThread:2122155 [sender.py:send():379] send: exit
+2025-01-06 22:17:39,742 INFO    SenderThread:2122155 [sender.py:send_exit():586] handling exit code: 0
+2025-01-06 22:17:39,742 INFO    SenderThread:2122155 [sender.py:send_exit():588] handling runtime: 256
+2025-01-06 22:17:39,742 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:39,742 INFO    SenderThread:2122155 [sender.py:send_exit():594] send defer
+2025-01-06 22:17:39,742 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:39,742 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-06 22:17:39,743 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:39,743 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-06 22:17:39,743 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 1
+2025-01-06 22:17:39,743 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:39,743 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-06 22:17:39,743 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:39,743 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-06 22:17:39,743 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 2
+2025-01-06 22:17:39,743 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:39,743 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-06 22:17:39,743 INFO    HandlerThread:2122155 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-06 22:17:39,744 INFO    HandlerThread:2122155 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-06 22:17:39,744 INFO    HandlerThread:2122155 [interfaces.py:finish():200] Joined disk monitor
+2025-01-06 22:17:39,744 DEBUG   SystemMonitor:2122155 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-06 22:17:39,744 DEBUG   SystemMonitor:2122155 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-06 22:17:39,780 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/gmfg8mam-media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json
+2025-01-06 22:17:39,794 INFO    HandlerThread:2122155 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-06 22:17:39,794 INFO    HandlerThread:2122155 [interfaces.py:finish():200] Joined memory monitor
+2025-01-06 22:17:39,794 INFO    HandlerThread:2122155 [interfaces.py:finish():200] Joined network monitor
+2025-01-06 22:17:39,795 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:39,795 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-06 22:17:39,795 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 3
+2025-01-06 22:17:39,795 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:17:39,795 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:39,795 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-06 22:17:39,795 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:39,795 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-06 22:17:39,795 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 4
+2025-01-06 22:17:39,796 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:39,796 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-06 22:17:40,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:40,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:17:42,124 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:43,124 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml
+2025-01-06 22:17:45,871 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:45,871 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:45,871 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-06 22:17:45,872 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:45,872 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-06 22:17:45,872 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 5
+2025-01-06 22:17:45,872 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:45,872 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:45,872 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-06 22:17:45,872 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:17:45,873 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:45,876 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:45,876 DEBUG   SenderThread:2122155 [sender.py:send():379] send: summary
+2025-01-06 22:17:45,876 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:45,876 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:45,876 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-06 22:17:45,876 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 6
+2025-01-06 22:17:45,876 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:45,877 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-06 22:17:45,877 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:45,877 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-06 22:17:45,877 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 7
+2025-01-06 22:17:45,877 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:45,877 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:45,877 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-06 22:17:45,877 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:45,877 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-06 22:17:46,125 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:17:46,743 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:49,237 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 8
+2025-01-06 22:17:49,237 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:49,237 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:49,237 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-06 22:17:49,238 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:49,238 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-06 22:17:49,238 INFO    SenderThread:2122155 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-06 22:17:49,240 INFO    SenderThread:2122155 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-06 22:17:49,242 INFO    SenderThread:2122155 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-06 22:17:49,256 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 9
+2025-01-06 22:17:49,257 DEBUG   SenderThread:2122155 [sender.py:send():379] send: artifact
+2025-01-06 22:17:49,257 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:49,259 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-06 22:17:49,503 INFO    SenderThread:2122155 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}}
+2025-01-06 22:17:49,503 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:49,503 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-06 22:17:49,503 INFO    SenderThread:2122155 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-06 22:17:49,744 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:50,126 INFO    SenderThread:2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:50,127 INFO    SenderThread:2122155 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files
+2025-01-06 22:17:50,127 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml config.yaml
+2025-01-06 22:17:50,127 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json wandb-summary.json
+2025-01-06 22:17:50,127 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/requirements.txt requirements.txt
+2025-01-06 22:17:50,130 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-metadata.json wandb-metadata.json
+2025-01-06 22:17:50,130 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0 events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log output.log
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/diff.patch diff.patch
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4 media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4 media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4 media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4
+2025-01-06 22:17:50,133 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4 media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4
+2025-01-06 22:17:50,133 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4 media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4
+2025-01-06 22:17:50,133 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 22:17:50,133 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json
+2025-01-06 22:17:50,134 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 10
+2025-01-06 22:17:50,134 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:50,135 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:50,136 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-06 22:17:50,136 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:50,136 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-06 22:17:50,136 INFO    SenderThread:2122155 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 22:17:50,303 INFO    wandb-upload_2:2122155 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml
+2025-01-06 22:17:50,312 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:17:50,320 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/requirements.txt
+2025-01-06 22:17:50,358 INFO    wandb-upload_4:2122155 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:50,413 INFO    wandb-upload_3:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/6f9yw2hd-events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:17:50,613 INFO    Thread-11 (_thread_body):2122155 [sender.py:transition_state():614] send defer: 11
+2025-01-06 22:17:50,614 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:50,614 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-06 22:17:50,614 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:50,614 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-06 22:17:50,614 INFO    SenderThread:2122155 [file_pusher.py:join():175] waiting for file pusher
+2025-01-06 22:17:50,615 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 12
+2025-01-06 22:17:50,615 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:50,615 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-06 22:17:50,615 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:50,616 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-06 22:17:50,616 INFO    SenderThread:2122155 [file_stream.py:finish():601] file stream finish called
+2025-01-06 22:17:50,744 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:50,802 INFO    SenderThread:2122155 [file_stream.py:finish():605] file stream finish is done
+2025-01-06 22:17:50,802 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 13
+2025-01-06 22:17:50,803 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:50,803 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:50,803 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-06 22:17:50,803 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:50,803 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-06 22:17:50,803 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 14
+2025-01-06 22:17:50,803 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:50,803 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-06 22:17:50,804 DEBUG   SenderThread:2122155 [sender.py:send():379] send: final
+2025-01-06 22:17:50,804 DEBUG   SenderThread:2122155 [sender.py:send():379] send: footer
+2025-01-06 22:17:50,804 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:50,804 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-06 22:17:50,804 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:50,805 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:50,805 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: server_info
+2025-01-06 22:17:50,805 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:50,805 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-06 22:17:50,805 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: server_info
+2025-01-06 22:17:50,806 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-06 22:17:50,808 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:50,858 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:50,858 INFO    MainThread:2122155 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-06 22:17:50,858 INFO    MainThread:2122155 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-06 22:17:50,859 INFO    MainThread:2122155 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-06 22:17:50,859 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-06 22:17:50,859 INFO    HandlerThread:2122155 [handler.py:finish():882] shutting down handler
+2025-01-06 22:17:51,806 INFO    WriterThread:2122155 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/run-9ig9megf.wandb
+2025-01-06 22:17:51,858 INFO    SenderThread:2122155 [sender.py:finish():1615] shutting down sender
+2025-01-06 22:17:51,858 INFO    SenderThread:2122155 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 22:17:51,858 INFO    SenderThread:2122155 [file_pusher.py:join():175] waiting for file pusher
diff --git a/9ig9megf/wandb/debug.log b/9ig9megf/wandb/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..019e9121d6485205980f42b35ed0dc217503a332
--- /dev/null
+++ b/9ig9megf/wandb/debug.log
@@ -0,0 +1,31 @@
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Configure stats pid to 2121985
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/settings
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/logs/debug.log
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/logs/debug-internal.log
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_init.py:init():569] calling init triggers
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf'}
+2025-01-06 22:13:22,741 INFO    MainThread:2121985 [wandb_init.py:init():619] starting backend
+2025-01-06 22:13:22,741 INFO    MainThread:2121985 [wandb_init.py:init():623] setting up manager
+2025-01-06 22:13:22,747 INFO    MainThread:2121985 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-06 22:13:22,749 INFO    MainThread:2121985 [wandb_init.py:init():631] backend started and connected
+2025-01-06 22:13:22,763 INFO    MainThread:2121985 [wandb_init.py:init():720] updated telemetry
+2025-01-06 22:13:22,769 INFO    MainThread:2121985 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-06 22:13:23,054 INFO    MainThread:2121985 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-06 22:13:23,100 INFO    MainThread:2121985 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-06 22:13:23,101 INFO    MainThread:2121985 [wandb_init.py:init():804] starting run threads in backend
+2025-01-06 22:13:23,467 INFO    MainThread:2121985 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-06 22:13:23,467 INFO    MainThread:2121985 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-06 22:13:23,467 INFO    MainThread:2121985 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-06 22:13:23,467 INFO    MainThread:2121985 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-06 22:13:23,469 INFO    MainThread:2121985 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-06 22:13:23,470 INFO    MainThread:2121985 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/tb, True
+2025-01-06 22:13:33,334 INFO    MainThread:2121985 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'}
+2025-01-06 22:17:51,860 WARNING MsgRouterThr:2121985 [router.py:message_loop():77] message_loop has been closed
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code/FISH/eval_robot.py b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e61d778421521cdd1c1fd43606f441e7abd00564
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml
@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 1
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1009_1'
+wandb_notes:
+  desc: null
+  value: 1009_1_restarted_3
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 5
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 1
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1736219602
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: lady
+left_book_slot:
+  desc: null
+  value: twodim
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/diff.patch b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/diff.patch
new file mode 100644
index 0000000000000000000000000000000000000000..43dd1a505c5d25a547a09b1d8e6dc31eb2a6a985
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/diff.patch
@@ -0,0 +1,63 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index d2afe29..101f9bd 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint}
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1002_0' 
++wandb_run_id: '1009_1' # seed 1
++# wandb_run_id: '1022_0' # dataset shuffle seed 1
++
++# RGBD+mask+contact(all ftrs)+act history
++# wandb_run_id: '1003_0' 
++# wandb_run_id: '1013_1' # seed 1
++# wandb_run_id: '1017_0' # dataset shuffle seed 1
+ 
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1007_0' 
+-wandb_run_id: '1922_1' # seed 1
++# wandb_run_id: '1922_1' # seed 1
+ # wandb_run_id: '1938_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1
+ # wandb_run_id: '1926_1' # seed 1
+ # wandb_run_id: '1930_0' # dataset shuffle seed 1
+ 
+-
+-
+ # RGBD+mask+contact(contact+DTC)+act history
+ # wandb_run_id: '1745_0'
+ 
++# all books, 4/20 demos per book
++
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+ 
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index c7eadce..e3c7dc1 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -161,8 +161,18 @@ run_id_list = [
+ run_id_list = [
+     # '1922_1',
+     # '1922_2',
+-    '1926_1',
+-    '1930_0',
++    # '1944_0',
++    # '1972_0',
++    # '1976_1',
++    # '1948_0',
++    # '1950_0',
++    # '1954_1',
++    '1002_0',
++    '1009_1',
++    '1022_0',
++    '1003_0',
++    '1013_1',
++    '1017_0',
+ ]
+ 
+ checkpoint_epoch = 12000
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]}
\ No newline at end of file
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json
new file mode 100644
index 0000000000000000000000000000000000000000..48ec72b039e2da82f6ab6cb3fe19cd956988e767
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json
@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[30, 30, 1.0, 0.9964111446341468, 1, 0.89405146233307, 1, 0.908258702524311, 1, 1, 1]]}
\ No newline at end of file
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4 b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1c63efc59d438bb63dae93fee450bf240520ecea
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44f7c849dd38a4c00ad495c118149a4e0b9bc9f0dcde5b0b2df02a50345eefef
+size 1128228
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4 b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..d0233c45b00a3bace8cc4b1a5d9576eb82eab306
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c32d361c2cac363e3ab2b1a5610d022314c98381ef4cc7e8ac72037370502f6
+size 1134779
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4 b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4ef69ab9875fba0d84665befd8d2b2ae1e8c094b
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4214c17fed8f2dd3c5b4f1652933de7dcf6275b143d806648990f82e86f77e2f
+size 1135935
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4 b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1677e899487ad0cf6212b5154dff27a5c2fbcbf5
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a87ee96aee6886a00c6c376db2338c44627415aa14577c6e00666d529ba4bbc
+size 1174830
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4 b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..f166054f207a9a2aa09dc143b533535eb110681d
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eed30eb0a971980a2bf81177202b399340800a8766d47fdba2febd8956f5f421
+size 1144795
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
new file mode 100644
index 0000000000000000000000000000000000000000..ecda33b1a9f495b3e9ae1c39fe2a2048480936f7
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
@@ -0,0 +1,272 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1736219613.335561]: resetting environment
+[INFO] [1736219613.341130]: cleared current plan
+[INFO] [1736219613.342273]: moving to home
+[INFO] [1736219614.344630]: reached home
+[INFO] [1736219614.346488]: reset action history
+[INFO] [1736219617.054506]: environment reset
+Starting episode 0
+[INFO] [1736219617.056580]: resetting environment
+[INFO] [1736219617.059882]: cleared current plan
+[INFO] [1736219617.060648]: moving to home
+[INFO] [1736219618.063645]: reached home
+[INFO] [1736219618.066374]: reset action history
+[INFO] [1736219620.774735]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1736219640.853969]: resetting environment
+[INFO] [1736219640.859764]: cleared current plan
+[INFO] [1736219640.860810]: moving to home
+[INFO] [1736219641.862812]: reached home
+[INFO] [1736219641.864084]: reset action history
+[INFO] [1736219644.570557]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/episode_rosbags/episode_0_2025-01-06-22-14-04.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2121985.log
+started roslaunch server http://158.130.50.37:37397/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2122287]
+process[print_text-2]: started with pid [2122288]
+process[rosbag_record-3]: started with pid [2122289]
+started bagging!
+[INFO] [1736219671.604391]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1736219672.350517]: Stored episode 1.
+[INFO] [1736219672.350773]: Saving video...
+[INFO] [1736219672.559524]: Video saved!
+Starting episode 1
+[INFO] [1736219672.570915]: resetting environment
+[INFO] [1736219672.576319]: cleared current plan
+[INFO] [1736219672.577221]: moving to home
+[INFO] [1736219676.179999]: reached home
+[INFO] [1736219676.180463]: reset action history
+[INFO] [1736219678.889580]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736219689.080960]: resetting environment
+[INFO] [1736219689.084717]: cleared current plan
+[INFO] [1736219689.084882]: moving to home
+[INFO] [1736219690.086824]: reached home
+[INFO] [1736219690.087252]: reset action history
+[INFO] [1736219692.793900]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/episode_rosbags/episode_1_2025-01-06-22-14-52.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2121985.log
+started roslaunch server http://158.130.50.37:43631/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2122400]
+process[print_text-5]: started with pid [2122401]
+process[rosbag_record-6]: started with pid [2122402]
+started bagging!
+For topic gripper_width: timestamp difference is -192797130 for nearest: 1736219700490736539 - target: 1736219700683533669 at idx 105
+For topic gripper_width: timestamp difference is -189140831 for nearest: 1736219705090780224 - target: 1736219705279921055 at idx 163
+[INFO] [1736219719.737607]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1736219720.517854]: Stored episode 2.
+[INFO] [1736219720.518194]: Saving video...
+[INFO] [1736219721.147486]: Video saved!
+Starting episode 2
+[INFO] [1736219721.184417]: resetting environment
+[INFO] [1736219721.194734]: cleared current plan
+[WARN] [1736219721.099916]: Plan exhausted
+[WARN] [1736219721.139464]: Plan exhausted
+[WARN] [1736219721.178866]: Plan exhausted
+[INFO] [1736219721.197052]: moving to home
+[INFO] [1736219724.809569]: reached home
+[INFO] [1736219724.813761]: reset action history
+[INFO] [1736219727.583335]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736219731.510272]: resetting environment
+[INFO] [1736219731.519870]: cleared current plan
+[INFO] [1736219731.520543]: moving to home
+[INFO] [1736219732.521268]: reached home
+[INFO] [1736219732.521759]: reset action history
+[INFO] [1736219735.229443]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/episode_rosbags/episode_2_2025-01-06-22-15-35.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2121985.log
+started roslaunch server http://158.130.50.37:42363/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [2122561]
+process[print_text-8]: started with pid [2122562]
+process[rosbag_record-9]: started with pid [2122563]
+started bagging!
+For topic gripper_width: timestamp difference is 102817029 for nearest: 1736219736757403344 - target: 1736219736654586315 at idx 22
+For topic gripper_width: timestamp difference is 104878724 for nearest: 1736219739357402861 - target: 1736219739252524137 at idx 63
+For topic gripper_width: timestamp difference is 106796074 for nearest: 1736219741757403660 - target: 1736219741650607586 at idx 93
+[INFO] [1736219762.332724]: Storing episode...
+[print_text-8] killing on exit[pub_text-7] killing on exit
+[rosbag_record-9] killing on exit
+[INFO] [1736219763.110806]: Stored episode 3.
+[INFO] [1736219763.115821]: Saving video...
+[INFO] [1736219763.340408]: Video saved!
+Starting episode 3
+[INFO] [1736219763.378185]: resetting environment
+[INFO] [1736219763.395144]: cleared current plan
+[INFO] [1736219763.396059]: moving to home
+[INFO] [1736219766.999710]: reached home
+[INFO] [1736219767.002456]: reset action history
+[INFO] [1736219769.737163]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736219771.252411]: resetting environment
+[INFO] [1736219771.273155]: cleared current plan
+[INFO] [1736219771.274935]: moving to home
+[INFO] [1736219772.281593]: reached home
+[INFO] [1736219772.283754]: reset action history
+[INFO] [1736219775.049206]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/episode_rosbags/episode_3_2025-01-06-22-16-15.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2121985.log
+started roslaunch server http://158.130.50.37:35345/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [2122696]
+process[print_text-11]: started with pid [2122697]
+process[rosbag_record-12]: started with pid [2122698]
+started bagging!
+For topic gripper_width: timestamp difference is -116647629 for nearest: 1736219794990736814 - target: 1736219795107384443 at idx 249
+[INFO] [1736219802.992603]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1736219803.739134]: Stored episode 4.
+[INFO] [1736219803.744666]: Saving video...
+[INFO] [1736219804.005052]: Video saved!
+Starting episode 4
+[INFO] [1736219804.067480]: resetting environment
+[INFO] [1736219804.093648]: cleared current plan
+[INFO] [1736219804.100384]: moving to home
+[INFO] [1736219807.709318]: reached home
+[INFO] [1736219807.714094]: reset action history
+[INFO] [1736219810.518888]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736219815.951962]: resetting environment
+[INFO] [1736219815.967272]: cleared current plan
+[INFO] [1736219815.973387]: moving to home
+[INFO] [1736219816.976501]: reached home
+[INFO] [1736219816.978682]: reset action history
+[INFO] [1736219819.761740]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/episode_rosbags/episode_4_2025-01-06-22-16-59.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2121985.log
+started roslaunch server http://158.130.50.37:41359/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [2122829]
+process[print_text-14]: started with pid [2122830]
+process[rosbag_record-15]: started with pid [2122831]
+started bagging!
+[INFO] [1736219846.949729]: Storing episode...
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit
+[print_text-14] killing on exit
+[WARN] [1736219848.300165]: Plan exhausted
+[WARN] [1736219848.339556]: Plan exhausted
+[INFO] [1736219847.737094]: Stored episode 5.
+[INFO] [1736219847.744877]: Saving video...
+[INFO] [1736219848.289339]: Video saved!
+[INFO] [1736219848.338176]: resetting environment
+[INFO] [1736219848.355612]: cleared current plan
+[INFO] [1736219848.356446]: moving to home
+[INFO] [1736219851.962925]: reached home
+[INFO] [1736219851.965210]: reset action history
+[INFO] [1736219854.821468]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+proceeding to start episode!
\ No newline at end of file
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/requirements.txt b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/requirements.txt
@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0
\ No newline at end of file
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-metadata.json b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-metadata.json
new file mode 100644
index 0000000000000000000000000000000000000000..18afe73c37d480323197ce54356bb7c656e90cfa
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-metadata.json
@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-07T03:13:23.124287",
+    "startedAt": "2025-01-07T03:13:22.738878",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3748.5340000000006,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4491.899,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4496.29,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.995,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 558.2445449829102
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
new file mode 100644
index 0000000000000000000000000000000000000000..e64b53a7732a2ef51b6cd4fd38c71b84362dde2a
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "44f7c849dd38a4c00ad495c118149a4e0b9bc9f0dcde5b0b2df02a50345eefef", "size": 1128228, "path": "media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4"}, "global_step": 665, "_timestamp": 1736219858.2501512, "_runtime": 255.50075006484985, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "1c32d361c2cac363e3ab2b1a5610d022314c98381ef4cc7e8ac72037370502f6", "size": 1134779, "path": "media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4"}, "eval/num_success": 5.0, "episode": 30.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "4214c17fed8f2dd3c5b4f1652933de7dcf6275b143d806648990f82e86f77e2f", "size": 1135935, "path": "media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "2a87ee96aee6886a00c6c376db2338c44627415aa14577c6e00666d529ba4bbc", "size": 1174830, "path": "media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "eed30eb0a971980a2bf81177202b399340800a8766d47fdba2febd8956f5f421", "size": 1144795, "path": "media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://bs8e4pbd5c4pfqm1jochr448odvm14mo6ar1vvof4pzqgplper3ytn5aj5sb56pjci9ublgqjx8aa9nflimxoxkpb6uan7eowlgf2q2s3at0oq272b220289mjvw5eb0/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ecybrwpe3dqvqvfoj8sosguvhufn2cqz3ml9b7zc2769xs2l20rhv9d4ugacteym2xft9u9ds0t0rvbspvmkblmbrlknkqcuuk30i5zxkex54s810vgseh4xbmriyfhp:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 30, "total_num_episodes": 30, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.9964111446341468, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.89405146233307, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.908258702524311, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 1, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c696f5811b0274780cec19cce17b1a03b91d0f7e00fc5d435a287eed68232c42", "size": 417, "artifact_path": "wandb-client-artifact://7nrif5iemxbn9lsh69tmcxclto9n6bhqm262f3ynezxonfloztxaixbdcpo9prfqgjrpj8fqk32llqypkifntyrtb0yqhmkz5ghmqrnfag898gd90b877lt1gqgtklqg/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ajewrnpvnglqzifudz9qyo315hbrk9p5t0dihu17bb75h5bocq1fwit12b82m1hzqz5irinaz7o1tr5ki01gg6r8zg38kijrv92dtjgprxga44jzvmtrxoyq8qapp4ts:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 256}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.996411144733429, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.8940514326095581, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9082586765289307, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 1.0, "eval/total_success_rate_ub_0.1": 1.0}
\ No newline at end of file
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/logs/debug-internal.log b/9ig9megf/wandb/run-20250106_221322-9ig9megf/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..c7988606b7e7669ab05c76277040cea71b90eda0
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/logs/debug-internal.log
@@ -0,0 +1,755 @@
+2025-01-06 22:13:22,748 INFO    StreamThr :2122155 [internal.py:wandb_internal():85] W&B internal server running at pid: 2122155, started at: 2025-01-06 22:13:22.748336
+2025-01-06 22:13:22,750 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status
+2025-01-06 22:13:22,763 INFO    WriterThread:2122155 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/run-9ig9megf.wandb
+2025-01-06 22:13:22,763 DEBUG   SenderThread:2122155 [sender.py:send():379] send: header
+2025-01-06 22:13:22,771 DEBUG   SenderThread:2122155 [sender.py:send():379] send: run
+2025-01-06 22:13:23,035 INFO    SenderThread:2122155 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files
+2025-01-06 22:13:23,035 INFO    SenderThread:2122155 [sender.py:_start_run_threads():1188] run started: 9ig9megf with start time 1736219602.749401
+2025-01-06 22:13:23,054 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: check_version
+2025-01-06 22:13:23,054 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: check_version
+2025-01-06 22:13:23,105 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: run_start
+2025-01-06 22:13:23,110 DEBUG   HandlerThread:2122155 [system_info.py:__init__():26] System info init
+2025-01-06 22:13:23,110 DEBUG   HandlerThread:2122155 [system_info.py:__init__():41] System info init done
+2025-01-06 22:13:23,111 INFO    HandlerThread:2122155 [system_monitor.py:start():194] Starting system monitor
+2025-01-06 22:13:23,111 INFO    SystemMonitor:2122155 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-06 22:13:23,111 INFO    HandlerThread:2122155 [system_monitor.py:probe():214] Collecting system info
+2025-01-06 22:13:23,111 INFO    SystemMonitor:2122155 [interfaces.py:start():188] Started cpu monitoring
+2025-01-06 22:13:23,112 INFO    SystemMonitor:2122155 [interfaces.py:start():188] Started disk monitoring
+2025-01-06 22:13:23,112 INFO    SystemMonitor:2122155 [interfaces.py:start():188] Started gpu monitoring
+2025-01-06 22:13:23,113 INFO    SystemMonitor:2122155 [interfaces.py:start():188] Started memory monitoring
+2025-01-06 22:13:23,113 INFO    SystemMonitor:2122155 [interfaces.py:start():188] Started network monitoring
+2025-01-06 22:13:23,124 DEBUG   HandlerThread:2122155 [system_info.py:probe():152] Probing system
+2025-01-06 22:13:23,125 DEBUG   HandlerThread:2122155 [system_info.py:_probe_git():137] Probing git
+2025-01-06 22:13:23,130 DEBUG   HandlerThread:2122155 [system_info.py:_probe_git():145] Probing git done
+2025-01-06 22:13:23,130 DEBUG   HandlerThread:2122155 [system_info.py:probe():200] Probing system done
+2025-01-06 22:13:23,130 DEBUG   HandlerThread:2122155 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:13:23.124287', 'startedAt': '2025-01-07T03:13:22.738878', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.5340000000006, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4491.899, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.29, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.995, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 558.2445449829102}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-06 22:13:23,130 INFO    HandlerThread:2122155 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-06 22:13:23,130 INFO    HandlerThread:2122155 [system_monitor.py:probe():227] Publishing system info
+2025-01-06 22:13:23,130 DEBUG   HandlerThread:2122155 [system_info.py:_save_code():44] Saving code
+2025-01-06 22:13:23,131 DEBUG   HandlerThread:2122155 [system_info.py:_save_code():67] Saving code done
+2025-01-06 22:13:23,132 DEBUG   HandlerThread:2122155 [system_info.py:_save_patches():84] Saving git patches
+2025-01-06 22:13:23,192 DEBUG   HandlerThread:2122155 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-06 22:13:23,193 INFO    HandlerThread:2122155 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-06 22:13:23,196 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:13:23,196 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-06 22:13:23,196 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-06 22:13:23,197 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-06 22:13:23,440 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/812ddovu-wandb-metadata.json
+2025-01-06 22:13:23,464 INFO    wandb-upload_2:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/8bsaca9a-diff.patch
+2025-01-06 22:13:23,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-06 22:13:23,467 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: python_packages
+2025-01-06 22:13:23,468 DEBUG   SenderThread:2122155 [sender.py:send():379] send: telemetry
+2025-01-06 22:13:23,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:13:23,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:13:23,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:23,470 INFO    HandlerThread:2122155 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/tb"
+  save: true
+}
+control {
+  relay_id: "d08939d596444ee6b2c634f165868670"
+}
+_info {
+  stream_id: "9ig9megf"
+}
+
+2025-01-06 22:13:23,470 DEBUG   HandlerThread:2122155 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-06 22:13:23,488 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/oh5ciig3-code/FISH/eval_robot.py
+2025-01-06 22:13:23,570 DEBUG   SenderThread:2122155 [sender.py:send():379] send: tbrecord
+2025-01-06 22:13:23,571 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:13:23,571 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0 with policy live
+2025-01-06 22:13:23,758 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/ig0mgqma-events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:13:24,036 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code/FISH/eval_robot.py
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/requirements.txt
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-metadata.json
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/diff.patch
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code
+2025-01-06 22:13:24,037 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code/FISH
+2025-01-06 22:13:24,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:25,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:26,036 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:26,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:27,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:28,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:28,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:29,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:30,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:31,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:32,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:33,334 DEBUG   SenderThread:2122155 [sender.py:send():379] send: config
+2025-01-06 22:13:33,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:33,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:34,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:35,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:36,039 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:36,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:37,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:38,040 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:38,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:13:38,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:13:38,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:38,535 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:39,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:40,040 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:40,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:41,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:42,041 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:13:42,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:43,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:44,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:44,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:45,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:46,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:47,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:48,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:49,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:49,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:50,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:51,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:52,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:53,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:13:53,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:13:53,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:54,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:54,563 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:13:55,044 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml
+2025-01-06 22:13:55,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:56,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:57,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:58,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:13:59,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:00,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:00,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:01,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:02,046 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:02,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:03,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:04,047 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:04,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:05,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:05,888 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:06,048 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:06,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:07,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:08,467 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:14:08,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:14:08,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:09,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:10,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:11,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:11,581 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:12,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:13,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:14,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:15,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:16,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:17,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:17,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:18,474 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:19,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:20,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:21,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:22,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:23,114 DEBUG   SystemMonitor:2122155 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-06 22:14:23,114 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:14:23,115 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:23,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:14:23,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:14:23,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:24,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:25,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:26,476 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:27,475 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:28,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:28,476 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:29,476 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:30,476 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:31,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:32,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:32,601 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:14:32,602 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:14:32,602 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4 with policy now
+2025-01-06 22:14:32,602 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:14:32,602 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:14:32,603 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:14:32,859 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/lil8d117-media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4
+2025-01-06 22:14:33,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4
+2025-01-06 22:14:33,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:14:33,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media
+2025-01-06 22:14:33,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval
+2025-01-06 22:14:33,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos
+2025-01-06 22:14:33,331 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/41hp5crh-media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4
+2025-01-06 22:14:33,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:33,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:34,055 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:34,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:35,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:36,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:37,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:38,056 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:38,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:14:38,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:14:38,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:38,539 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:39,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:40,057 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:40,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:41,478 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:42,478 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:43,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:44,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:44,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:45,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:46,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:47,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:48,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:49,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:50,061 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:50,061 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:14:50,087 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:50,304 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/85bojlbt-events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:14:50,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:51,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:52,061 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:52,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:53,115 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:14:53,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:14:53,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:14:53,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:54,062 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:14:54,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:55,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:55,527 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:14:56,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:57,480 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:58,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:14:59,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:00,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:00,768 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:01,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:02,065 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:02,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:03,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:04,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:05,481 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:06,413 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:06,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:07,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:08,067 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:08,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:15:08,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:15:08,513 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:09,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:10,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:11,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:11,593 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:12,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:13,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:14,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:15,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:16,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:17,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:17,482 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:18,483 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:19,483 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:20,483 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:21,174 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:15:21,174 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4 with policy now
+2025-01-06 22:15:21,178 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:15:21,179 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:15:21,179 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:15:21,179 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:15:21,426 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/ty8szote-media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4
+2025-01-06 22:15:21,485 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:22,071 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:22,071 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:15:22,071 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4
+2025-01-06 22:15:22,072 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval
+2025-01-06 22:15:22,426 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/9vjyf2vq-media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4
+2025-01-06 22:15:22,484 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:23,115 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:15:23,115 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:23,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:15:23,469 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:15:23,513 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:24,072 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:24,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:25,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:26,072 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:26,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:27,486 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:28,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:28,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:29,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:30,074 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:30,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:31,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:31,932 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:15:31,932 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:15:31,932 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:15:32,075 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:15:32,075 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:15:32,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:33,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:33,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:34,075 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:34,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:35,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:36,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:37,487 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:38,077 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:38,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:15:38,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:15:38,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:38,535 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:39,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:40,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:41,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:42,078 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:42,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:43,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:44,079 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:15:44,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:44,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:45,488 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:46,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:47,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:48,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:49,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:49,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:50,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:51,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:52,489 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:53,116 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:15:53,468 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:15:53,468 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:15:53,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:54,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:54,601 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:15:55,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:56,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:57,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:58,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:15:59,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:00,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:00,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:01,490 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:02,491 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:03,375 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:16:03,375 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4 with policy now
+2025-01-06 22:16:03,377 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:16:03,378 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:16:03,378 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:16:03,379 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:16:03,491 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:03,671 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/2m9ytad9-media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4
+2025-01-06 22:16:04,087 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:04,087 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:16:04,087 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4
+2025-01-06 22:16:04,087 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval
+2025-01-06 22:16:04,385 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/p42gscn4-media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4
+2025-01-06 22:16:04,492 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:05,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:05,497 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:06,087 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:06,500 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:07,501 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:08,088 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:08,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:16:08,471 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:16:08,513 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:09,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:10,505 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:10,781 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:11,506 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:11,757 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:16:11,757 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:16:11,775 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:16:12,090 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:12,090 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:16:12,090 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:16:12,329 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/832f55tw-events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:16:12,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:13,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:14,090 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:14,512 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:15,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:16,003 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:16,091 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:16,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:17,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:18,092 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:18,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:19,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:20,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:21,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:21,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:22,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:23,116 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:16:23,469 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:16:23,470 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:16:23,513 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:24,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:25,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:26,507 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:26,599 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:27,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:28,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:29,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:30,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:31,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:32,472 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:32,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:33,508 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:34,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:35,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:36,098 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:36,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:37,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:38,229 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:38,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:16:38,470 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:16:38,517 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:39,510 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:40,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:41,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:42,509 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:43,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:43,516 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:44,057 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:16:44,057 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4 with policy now
+2025-01-06 22:16:44,061 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:16:44,061 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:16:44,061 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:16:44,062 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:16:44,100 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:44,101 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:16:44,101 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4
+2025-01-06 22:16:44,101 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval
+2025-01-06 22:16:44,312 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/6xeimi59-media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4
+2025-01-06 22:16:44,514 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:44,592 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/3xd02x84-media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4
+2025-01-06 22:16:45,517 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:46,101 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:46,514 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:47,518 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:48,519 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:49,477 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:49,523 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:50,102 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:50,519 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:51,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:52,103 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:52,520 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:53,117 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:16:53,473 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:16:53,473 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:16:53,523 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:54,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:54,539 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:16:55,523 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:56,104 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:16:56,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:56,813 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:16:56,813 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:16:56,813 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:16:57,105 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:16:57,526 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:58,105 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:16:58,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:59,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:16:59,762 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:00,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:01,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:02,107 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:02,524 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:03,528 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:04,528 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:05,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:05,528 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:06,528 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:07,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:08,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:17:08,470 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:17:08,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:09,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:10,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:10,536 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:11,529 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:12,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:13,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:14,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:15,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:16,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:16,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:17,530 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:18,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:19,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:20,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:21,479 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:21,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:22,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:23,117 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:17:23,470 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:17:23,470 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:17:23,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:24,531 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:25,532 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:26,532 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:26,539 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:27,536 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:28,118 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:28,333 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:17:28,333 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4 with policy now
+2025-01-06 22:17:28,337 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:17:28,338 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:17:28,338 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:28,338 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:28,532 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:28,585 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/17fgcz54-media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4
+2025-01-06 22:17:29,119 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:17:29,119 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4
+2025-01-06 22:17:29,119 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval
+2025-01-06 22:17:29,409 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/uh1fc3so-media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4
+2025-01-06 22:17:29,534 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:30,119 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:30,537 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:31,542 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:31,963 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:32,544 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:33,551 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:34,120 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:34,548 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:35,546 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:36,121 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:36,552 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:37,320 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:37,550 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:37,775 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 22:17:37,775 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 22:17:37,947 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:17:38,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:38,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:17:38,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 22:17:38,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media
+2025-01-06 22:17:38,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table
+2025-01-06 22:17:38,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval
+2025-01-06 22:17:38,336 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/k02egdlf-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 22:17:38,346 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-06 22:17:38,355 INFO    wandb-upload_2:2122155 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplup4j1go
+2025-01-06 22:17:38,358 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 22:17:38,360 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/gylawia1-events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:17:38,471 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 22:17:38,547 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:38,744 INFO    SenderThread:2122155 [sender.py:send_request_log_artifact():1525] logged artifact run-9ig9megf-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU2MDY2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDczNTY1Nw==', 'latestArtifact': None}}
+2025-01-06 22:17:38,744 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:17:38,744 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now
+2025-01-06 22:17:38,745 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:17:38,745 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,745 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,746 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:17:38,746 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,749 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,749 DEBUG   SenderThread:2122155 [sender.py:send():379] send: telemetry
+2025-01-06 22:17:38,749 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,750 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,750 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,750 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,750 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,751 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,751 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,751 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,751 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,752 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,752 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,752 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,752 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,753 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,753 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,753 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,753 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,754 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,754 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,754 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,754 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:38,755 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:38,755 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: log_artifact
+2025-01-06 22:17:38,991 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/oa86qmqk-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 22:17:39,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:17:39,122 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:17:39,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json
+2025-01-06 22:17:39,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval
+2025-01-06 22:17:39,132 INFO    wandb-upload_1:2122155 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpeqtnnbg3
+2025-01-06 22:17:39,306 INFO    wandb-upload_2:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/m76450mq-media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json
+2025-01-06 22:17:39,535 INFO    SenderThread:2122155 [sender.py:send_request_log_artifact():1525] logged artifact run-9ig9megf-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU2MDY4MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDczNTY2MA==', 'latestArtifact': None}}
+2025-01-06 22:17:39,535 DEBUG   SenderThread:2122155 [sender.py:send():379] send: files
+2025-01-06 22:17:39,536 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json with policy now
+2025-01-06 22:17:39,536 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:17:39,536 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:39,537 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:39,537 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 22:17:39,547 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:39,741 DEBUG   SenderThread:2122155 [sender.py:send():379] send: exit
+2025-01-06 22:17:39,742 INFO    SenderThread:2122155 [sender.py:send_exit():586] handling exit code: 0
+2025-01-06 22:17:39,742 INFO    SenderThread:2122155 [sender.py:send_exit():588] handling runtime: 256
+2025-01-06 22:17:39,742 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:39,742 INFO    SenderThread:2122155 [sender.py:send_exit():594] send defer
+2025-01-06 22:17:39,742 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:39,742 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-06 22:17:39,743 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:39,743 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-06 22:17:39,743 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 1
+2025-01-06 22:17:39,743 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:39,743 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-06 22:17:39,743 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:39,743 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-06 22:17:39,743 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 2
+2025-01-06 22:17:39,743 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:39,743 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-06 22:17:39,743 INFO    HandlerThread:2122155 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-06 22:17:39,744 INFO    HandlerThread:2122155 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-06 22:17:39,744 INFO    HandlerThread:2122155 [interfaces.py:finish():200] Joined disk monitor
+2025-01-06 22:17:39,744 DEBUG   SystemMonitor:2122155 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-06 22:17:39,744 DEBUG   SystemMonitor:2122155 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-06 22:17:39,780 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/gmfg8mam-media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json
+2025-01-06 22:17:39,794 INFO    HandlerThread:2122155 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-06 22:17:39,794 INFO    HandlerThread:2122155 [interfaces.py:finish():200] Joined memory monitor
+2025-01-06 22:17:39,794 INFO    HandlerThread:2122155 [interfaces.py:finish():200] Joined network monitor
+2025-01-06 22:17:39,795 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:39,795 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-06 22:17:39,795 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 3
+2025-01-06 22:17:39,795 DEBUG   SenderThread:2122155 [sender.py:send():379] send: stats
+2025-01-06 22:17:39,795 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:39,795 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-06 22:17:39,795 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:39,795 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-06 22:17:39,795 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 4
+2025-01-06 22:17:39,796 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:39,796 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-06 22:17:40,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:40,123 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:17:42,124 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:43,124 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml
+2025-01-06 22:17:45,871 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:45,871 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:45,871 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-06 22:17:45,872 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:45,872 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-06 22:17:45,872 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 5
+2025-01-06 22:17:45,872 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:45,872 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:45,872 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-06 22:17:45,872 DEBUG   SenderThread:2122155 [sender.py:send():379] send: history
+2025-01-06 22:17:45,873 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 22:17:45,876 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:45,876 DEBUG   SenderThread:2122155 [sender.py:send():379] send: summary
+2025-01-06 22:17:45,876 INFO    SenderThread:2122155 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 22:17:45,876 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:45,876 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-06 22:17:45,876 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 6
+2025-01-06 22:17:45,876 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:45,877 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-06 22:17:45,877 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:45,877 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-06 22:17:45,877 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 7
+2025-01-06 22:17:45,877 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 22:17:45,877 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:45,877 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-06 22:17:45,877 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:45,877 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-06 22:17:46,125 INFO    Thread-12 :2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:17:46,743 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:49,237 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 8
+2025-01-06 22:17:49,237 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:49,237 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:49,237 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-06 22:17:49,238 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:49,238 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-06 22:17:49,238 INFO    SenderThread:2122155 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-06 22:17:49,240 INFO    SenderThread:2122155 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-06 22:17:49,242 INFO    SenderThread:2122155 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-06 22:17:49,256 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 9
+2025-01-06 22:17:49,257 DEBUG   SenderThread:2122155 [sender.py:send():379] send: artifact
+2025-01-06 22:17:49,257 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:49,259 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-06 22:17:49,503 INFO    SenderThread:2122155 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}}
+2025-01-06 22:17:49,503 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:49,503 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-06 22:17:49,503 INFO    SenderThread:2122155 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-06 22:17:49,744 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:50,126 INFO    SenderThread:2122155 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:50,127 INFO    SenderThread:2122155 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files
+2025-01-06 22:17:50,127 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml config.yaml
+2025-01-06 22:17:50,127 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json wandb-summary.json
+2025-01-06 22:17:50,127 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/requirements.txt requirements.txt
+2025-01-06 22:17:50,130 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-metadata.json wandb-metadata.json
+2025-01-06 22:17:50,130 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0 events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log output.log
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/diff.patch diff.patch
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4 media/videos/eval/0_eval_0_44f7c849dd38a4c00ad4.mp4
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4 media/videos/eval/1_eval_1_1c32d361c2cac363e3ab.mp4
+2025-01-06 22:17:50,132 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4 media/videos/eval/2_eval_2_4214c17fed8f2dd3c5b4.mp4
+2025-01-06 22:17:50,133 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4 media/videos/eval/3_eval_3_2a87ee96aee6886a00c6.mp4
+2025-01-06 22:17:50,133 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4 media/videos/eval/4_eval_4_eed30eb0a971980a2bf8.mp4
+2025-01-06 22:17:50,133 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-06 22:17:50,133 INFO    SenderThread:2122155 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json media/table/eval/total_success_rate_ci_6_c696f5811b0274780cec.table.json
+2025-01-06 22:17:50,134 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 10
+2025-01-06 22:17:50,134 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:50,135 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:50,136 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-06 22:17:50,136 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:50,136 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-06 22:17:50,136 INFO    SenderThread:2122155 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 22:17:50,303 INFO    wandb-upload_2:2122155 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/config.yaml
+2025-01-06 22:17:50,312 INFO    wandb-upload_1:2122155 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/wandb-summary.json
+2025-01-06 22:17:50,320 INFO    wandb-upload_0:2122155 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/requirements.txt
+2025-01-06 22:17:50,358 INFO    wandb-upload_4:2122155 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/files/output.log
+2025-01-06 22:17:50,413 INFO    wandb-upload_3:2122155 [upload_job.py:push():130] Uploaded file /tmp/tmpza8aoq14wandb/6f9yw2hd-events.out.tfevents.1736219603.leonmkim-ROG-Strix-G15CS-G15CS.2121985.0
+2025-01-06 22:17:50,613 INFO    Thread-11 (_thread_body):2122155 [sender.py:transition_state():614] send defer: 11
+2025-01-06 22:17:50,614 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:50,614 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-06 22:17:50,614 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:50,614 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-06 22:17:50,614 INFO    SenderThread:2122155 [file_pusher.py:join():175] waiting for file pusher
+2025-01-06 22:17:50,615 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 12
+2025-01-06 22:17:50,615 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:50,615 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-06 22:17:50,615 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:50,616 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-06 22:17:50,616 INFO    SenderThread:2122155 [file_stream.py:finish():601] file stream finish called
+2025-01-06 22:17:50,744 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:50,802 INFO    SenderThread:2122155 [file_stream.py:finish():605] file stream finish is done
+2025-01-06 22:17:50,802 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 13
+2025-01-06 22:17:50,803 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:50,803 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:50,803 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-06 22:17:50,803 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:50,803 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-06 22:17:50,803 INFO    SenderThread:2122155 [sender.py:transition_state():614] send defer: 14
+2025-01-06 22:17:50,803 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 22:17:50,803 INFO    HandlerThread:2122155 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-06 22:17:50,804 DEBUG   SenderThread:2122155 [sender.py:send():379] send: final
+2025-01-06 22:17:50,804 DEBUG   SenderThread:2122155 [sender.py:send():379] send: footer
+2025-01-06 22:17:50,804 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: defer
+2025-01-06 22:17:50,804 INFO    SenderThread:2122155 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-06 22:17:50,804 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:50,805 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:50,805 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: server_info
+2025-01-06 22:17:50,805 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 22:17:50,805 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-06 22:17:50,805 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: server_info
+2025-01-06 22:17:50,806 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-06 22:17:50,808 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 22:17:50,858 DEBUG   SenderThread:2122155 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 22:17:50,858 INFO    MainThread:2122155 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-06 22:17:50,858 INFO    MainThread:2122155 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-06 22:17:50,859 INFO    MainThread:2122155 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-06 22:17:50,859 DEBUG   HandlerThread:2122155 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-06 22:17:50,859 INFO    HandlerThread:2122155 [handler.py:finish():882] shutting down handler
+2025-01-06 22:17:51,806 INFO    WriterThread:2122155 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/run-9ig9megf.wandb
+2025-01-06 22:17:51,858 INFO    SenderThread:2122155 [sender.py:finish():1615] shutting down sender
+2025-01-06 22:17:51,858 INFO    SenderThread:2122155 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 22:17:51,858 INFO    SenderThread:2122155 [file_pusher.py:join():175] waiting for file pusher
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/logs/debug.log b/9ig9megf/wandb/run-20250106_221322-9ig9megf/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..019e9121d6485205980f42b35ed0dc217503a332
--- /dev/null
+++ b/9ig9megf/wandb/run-20250106_221322-9ig9megf/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Configure stats pid to 2121985
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/settings
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/logs/debug.log
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/wandb/run-20250106_221322-9ig9megf/logs/debug-internal.log
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_init.py:init():569] calling init triggers
+2025-01-06 22:13:22,740 INFO    MainThread:2121985 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf'}
+2025-01-06 22:13:22,741 INFO    MainThread:2121985 [wandb_init.py:init():619] starting backend
+2025-01-06 22:13:22,741 INFO    MainThread:2121985 [wandb_init.py:init():623] setting up manager
+2025-01-06 22:13:22,747 INFO    MainThread:2121985 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-06 22:13:22,749 INFO    MainThread:2121985 [wandb_init.py:init():631] backend started and connected
+2025-01-06 22:13:22,763 INFO    MainThread:2121985 [wandb_init.py:init():720] updated telemetry
+2025-01-06 22:13:22,769 INFO    MainThread:2121985 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-06 22:13:23,054 INFO    MainThread:2121985 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-06 22:13:23,100 INFO    MainThread:2121985 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-06 22:13:23,101 INFO    MainThread:2121985 [wandb_init.py:init():804] starting run threads in backend
+2025-01-06 22:13:23,467 INFO    MainThread:2121985 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-06 22:13:23,467 INFO    MainThread:2121985 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-06 22:13:23,467 INFO    MainThread:2121985 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-06 22:13:23,467 INFO    MainThread:2121985 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-06 22:13:23,469 INFO    MainThread:2121985 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-06 22:13:23,470 INFO    MainThread:2121985 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/9ig9megf/tb, True
+2025-01-06 22:13:33,334 INFO    MainThread:2121985 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'}
+2025-01-06 22:17:51,860 WARNING MsgRouterThr:2121985 [router.py:message_loop():77] message_loop has been closed
diff --git a/9ig9megf/wandb/run-20250106_221322-9ig9megf/run-9ig9megf.wandb b/9ig9megf/wandb/run-20250106_221322-9ig9megf/run-9ig9megf.wandb
new file mode 100644
index 0000000000000000000000000000000000000000..a52e947827ed2dd2dbe0fa086d5c119d8bfc02e4
Binary files /dev/null and b/9ig9megf/wandb/run-20250106_221322-9ig9megf/run-9ig9megf.wandb differ
diff --git a/9jw2ghih/episode_rosbags/episode_0_2025-01-06-20-41-14.bag b/9jw2ghih/episode_rosbags/episode_0_2025-01-06-20-41-14.bag
new file mode 100644
index 0000000000000000000000000000000000000000..ff80ca2879285563ad0e0dfd25bea13c275b089a
--- /dev/null
+++ b/9jw2ghih/episode_rosbags/episode_0_2025-01-06-20-41-14.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f97fb8ba2ef1bc50f59d62ab36e430b4e91b8e5fc1220c1ff13c1c6b7fb36f1
+size 1409587076
diff --git a/9jw2ghih/episode_rosbags/episode_1_2025-01-06-20-41-55.bag b/9jw2ghih/episode_rosbags/episode_1_2025-01-06-20-41-55.bag
new file mode 100644
index 0000000000000000000000000000000000000000..7d671a66b656317112e12d032a3d2c991e70b51a
--- /dev/null
+++ b/9jw2ghih/episode_rosbags/episode_1_2025-01-06-20-41-55.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cbab9b685a34cb3dc7d2037de77482ae4c0c49a2ee4382dc80fa5f893b3475c
+size 1412128971
diff --git a/9jw2ghih/episode_rosbags/episode_2_2025-01-06-20-42-37.bag b/9jw2ghih/episode_rosbags/episode_2_2025-01-06-20-42-37.bag
new file mode 100644
index 0000000000000000000000000000000000000000..4c070e34d7f85a8375ce72cb69035d0b08dfb6c7
--- /dev/null
+++ b/9jw2ghih/episode_rosbags/episode_2_2025-01-06-20-42-37.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6abb7c28d3a4bc4bcb1383ac4cc3ed336c3f5a826619f78d9978408b50b38c3a
+size 1411025609
diff --git a/9jw2ghih/episode_rosbags/episode_3_2025-01-06-20-43-18.bag b/9jw2ghih/episode_rosbags/episode_3_2025-01-06-20-43-18.bag
new file mode 100644
index 0000000000000000000000000000000000000000..1e12330a29f31cf10b81b7685f5b65b4e7c1c448
--- /dev/null
+++ b/9jw2ghih/episode_rosbags/episode_3_2025-01-06-20-43-18.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:836334a957a0e4b9a0887ad98158d22bbb0f688cb8390c319a28ded26facb7c2
+size 1409778561
diff --git a/i6xqd72y/.hydra/config.yaml b/i6xqd72y/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab
--- /dev/null
+++ b/i6xqd72y/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1009_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/i6xqd72y/.hydra/hydra.yaml b/i6xqd72y/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b4bfc8b5fbf95cefef5317ec729a7dfb42042a21
--- /dev/null
+++ b/i6xqd72y/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/i6xqd72y
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/i6xqd72y/.hydra/overrides.yaml b/i6xqd72y/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/i6xqd72y/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/i6xqd72y/eval_policy.log b/i6xqd72y/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..4e09a6599f1cfee7334b9feb264d24da7f586035
--- /dev/null
+++ b/i6xqd72y/eval_policy.log
@@ -0,0 +1,15 @@
+[2025-01-07 20:44:42,853][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-07 20:44:42,856][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-07 20:44:46,696][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+
diff --git a/nyyc47dd/episode_rosbags/episode_0_2025-01-07-17-10-04.bag b/nyyc47dd/episode_rosbags/episode_0_2025-01-07-17-10-04.bag
new file mode 100644
index 0000000000000000000000000000000000000000..475d52796117c7815be611ea9afbbbdabc0c523d
--- /dev/null
+++ b/nyyc47dd/episode_rosbags/episode_0_2025-01-07-17-10-04.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aac40d11400a520a100dc2044afc6a3433dc4f837bc99f352d997b366f83014d
+size 1414495941
diff --git a/nyyc47dd/episode_rosbags/episode_2_2025-01-07-17-11-29.bag b/nyyc47dd/episode_rosbags/episode_2_2025-01-07-17-11-29.bag
new file mode 100644
index 0000000000000000000000000000000000000000..b71da2fb08b6cbfe3eec95d7f705a7b11f7c2c09
--- /dev/null
+++ b/nyyc47dd/episode_rosbags/episode_2_2025-01-07-17-11-29.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a566564d05e77c48a2f8e822033e06c24fa6975f3a28aa4bdcddd68a521c7bce
+size 1412413584
diff --git a/nyyc47dd/episode_rosbags/episode_3_2025-01-07-17-12-13.bag b/nyyc47dd/episode_rosbags/episode_3_2025-01-07-17-12-13.bag
new file mode 100644
index 0000000000000000000000000000000000000000..47ae6c82b48405c1c52c63369675895e9f5df439
--- /dev/null
+++ b/nyyc47dd/episode_rosbags/episode_3_2025-01-07-17-12-13.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b25511aa04e63dffee6083e31efbae3a699a4441ebc78d00b7d8daf6d331c6a7
+size 1411729558
diff --git a/nyyc47dd/episode_rosbags/episode_4_2025-01-07-17-12-54.bag b/nyyc47dd/episode_rosbags/episode_4_2025-01-07-17-12-54.bag
new file mode 100644
index 0000000000000000000000000000000000000000..d12d92ac844bcd708dd0d478c3adc2ac163088d3
--- /dev/null
+++ b/nyyc47dd/episode_rosbags/episode_4_2025-01-07-17-12-54.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5a4145656d192287e52d2df0b0575c59228d11c1da7960da8c506355028807d
+size 1410590260
diff --git a/o9qcewu3/episode_rosbags/cam_tf_world.npy b/o9qcewu3/episode_rosbags/cam_tf_world.npy
new file mode 100644
index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a
--- /dev/null
+++ b/o9qcewu3/episode_rosbags/cam_tf_world.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256
diff --git a/o9qcewu3/eval_robot.log b/o9qcewu3/eval_robot.log
new file mode 100644
index 0000000000000000000000000000000000000000..df92593c1c83f00c24f3be9feddd010665ce970e
--- /dev/null
+++ b/o9qcewu3/eval_robot.log
@@ -0,0 +1,12 @@
+[2025-01-07 20:49:48,449][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-07 20:49:48,452][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
diff --git a/w3dz1sd1/.hydra/config.yaml b/w3dz1sd1/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab
--- /dev/null
+++ b/w3dz1sd1/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1009_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/w3dz1sd1/.hydra/hydra.yaml b/w3dz1sd1/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..61996674ed7feb98d594d1267cfcd2c3382a4354
--- /dev/null
+++ b/w3dz1sd1/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/w3dz1sd1/.hydra/overrides.yaml b/w3dz1sd1/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/w3dz1sd1/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/w3dz1sd1/episode_rosbags/episode_0_2025-01-07-17-46-27.bag b/w3dz1sd1/episode_rosbags/episode_0_2025-01-07-17-46-27.bag
new file mode 100644
index 0000000000000000000000000000000000000000..d57156b95112bb462fa0173a53978c8ddb0cf199
--- /dev/null
+++ b/w3dz1sd1/episode_rosbags/episode_0_2025-01-07-17-46-27.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71ba0cb8ee97148399fda5c1e15c451ff4ae8f08fda94972ebf756a1bcadb9bd
+size 1411492345
diff --git a/w3dz1sd1/episode_rosbags/episode_1_2025-01-07-17-47-21.bag b/w3dz1sd1/episode_rosbags/episode_1_2025-01-07-17-47-21.bag
new file mode 100644
index 0000000000000000000000000000000000000000..023f27da8f058a5af7ca092e72a1c1fa9e04ead7
--- /dev/null
+++ b/w3dz1sd1/episode_rosbags/episode_1_2025-01-07-17-47-21.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c6ebc635319a5d4f7d30f13639704ec261998774b1178b1a57224fe242428b8
+size 1412735308
diff --git a/w3dz1sd1/episode_rosbags/episode_2_2025-01-07-17-48-01.bag b/w3dz1sd1/episode_rosbags/episode_2_2025-01-07-17-48-01.bag
new file mode 100644
index 0000000000000000000000000000000000000000..b5e14723e98b61556e12e73e2b34f725d21e21e6
--- /dev/null
+++ b/w3dz1sd1/episode_rosbags/episode_2_2025-01-07-17-48-01.bag
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b597edbfd13a13b569d88d3c5aee7641e0234a1a9453f269255a0115901bc03e
+size 1413634076
diff --git a/w3dz1sd1/eval_video/0_eval.mp4 b/w3dz1sd1/eval_video/0_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ff873c8af303e057354f62cb3fc966b1a151a2a9
--- /dev/null
+++ b/w3dz1sd1/eval_video/0_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e4ad6e0af44158052246a0a1373fdcc2d70c928fdb4e3f0cb15cde8937bf7a6
+size 1145682
diff --git a/w3dz1sd1/eval_video/1_eval.mp4 b/w3dz1sd1/eval_video/1_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5adcb78d487dc2e6d1fc4bfd8360f57a23e53a5b
--- /dev/null
+++ b/w3dz1sd1/eval_video/1_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f499daa5972d95524f38e47699f47dbad979fd847f1d14020134898a7f3f7895
+size 1137042
diff --git a/w3dz1sd1/eval_video/2_eval.mp4 b/w3dz1sd1/eval_video/2_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..1a0ab87511bb9163f9a0ba04676996888c1d3c95
--- /dev/null
+++ b/w3dz1sd1/eval_video/2_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77080ebc0cccad9e8ca80aae9493c38fe600db1f954e05dd099fb9bb3c7aa7a3
+size 1147997
diff --git a/w3dz1sd1/eval_video/3_eval.mp4 b/w3dz1sd1/eval_video/3_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..0f5839e2a090362e7c28438aa0aaeae3a2e2ba09
--- /dev/null
+++ b/w3dz1sd1/eval_video/3_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea44f1493fcbb8437821d3a0c5910e6052dd27bda35f92dfdb93658bec5efbbf
+size 1156666
diff --git a/w3dz1sd1/eval_video/4_eval.mp4 b/w3dz1sd1/eval_video/4_eval.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..b90365762ac157d237556c8214b567fb54d4d327
--- /dev/null
+++ b/w3dz1sd1/eval_video/4_eval.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cadcc71b7662da48835a75b4c73eee2daf7d0a8e844c5da1154538d12f93b337
+size 1160393
diff --git a/w3dz1sd1/tb/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 b/w3dz1sd1/tb/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
new file mode 100644
index 0000000000000000000000000000000000000000..0509275e5cac3e68ab41ccd05d5994802f79762f
--- /dev/null
+++ b/w3dz1sd1/tb/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0abb092100ff116679e9b93b152b14be105bff6c6feb9655e1de3f132070ba5e
+size 6539
diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code/FISH/eval_robot.py b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code/FISH/eval_robot.py
new file mode 100644
index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710
--- /dev/null
+++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code/FISH/eval_robot.py
@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()
diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/logs/debug-internal.log b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/logs/debug-internal.log
new file mode 100644
index 0000000000000000000000000000000000000000..4569e918dd2df7c12e6556dfbc849d79819bf33e
--- /dev/null
+++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/logs/debug-internal.log
@@ -0,0 +1,780 @@
+2025-01-07 17:45:46,520 INFO    StreamThr :2202979 [internal.py:wandb_internal():85] W&B internal server running at pid: 2202979, started at: 2025-01-07 17:45:46.520389
+2025-01-07 17:45:46,521 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status
+2025-01-07 17:45:46,533 INFO    WriterThread:2202979 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/run-w3dz1sd1.wandb
+2025-01-07 17:45:46,533 DEBUG   SenderThread:2202979 [sender.py:send():379] send: header
+2025-01-07 17:45:46,541 DEBUG   SenderThread:2202979 [sender.py:send():379] send: run
+2025-01-07 17:45:46,844 INFO    SenderThread:2202979 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files
+2025-01-07 17:45:46,844 INFO    SenderThread:2202979 [sender.py:_start_run_threads():1188] run started: w3dz1sd1 with start time 1736289946.520927
+2025-01-07 17:45:46,859 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: check_version
+2025-01-07 17:45:46,859 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: check_version
+2025-01-07 17:45:46,905 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: run_start
+2025-01-07 17:45:46,910 DEBUG   HandlerThread:2202979 [system_info.py:__init__():26] System info init
+2025-01-07 17:45:46,910 DEBUG   HandlerThread:2202979 [system_info.py:__init__():41] System info init done
+2025-01-07 17:45:46,910 INFO    HandlerThread:2202979 [system_monitor.py:start():194] Starting system monitor
+2025-01-07 17:45:46,910 INFO    SystemMonitor:2202979 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-07 17:45:46,910 INFO    HandlerThread:2202979 [system_monitor.py:probe():214] Collecting system info
+2025-01-07 17:45:46,911 INFO    SystemMonitor:2202979 [interfaces.py:start():188] Started cpu monitoring
+2025-01-07 17:45:46,912 INFO    SystemMonitor:2202979 [interfaces.py:start():188] Started disk monitoring
+2025-01-07 17:45:46,912 INFO    SystemMonitor:2202979 [interfaces.py:start():188] Started gpu monitoring
+2025-01-07 17:45:46,912 INFO    SystemMonitor:2202979 [interfaces.py:start():188] Started memory monitoring
+2025-01-07 17:45:46,913 INFO    SystemMonitor:2202979 [interfaces.py:start():188] Started network monitoring
+2025-01-07 17:45:46,923 DEBUG   HandlerThread:2202979 [system_info.py:probe():152] Probing system
+2025-01-07 17:45:46,925 DEBUG   HandlerThread:2202979 [system_info.py:_probe_git():137] Probing git
+2025-01-07 17:45:46,929 DEBUG   HandlerThread:2202979 [system_info.py:_probe_git():145] Probing git done
+2025-01-07 17:45:46,929 DEBUG   HandlerThread:2202979 [system_info.py:probe():200] Probing system done
+2025-01-07 17:45:46,929 DEBUG   HandlerThread:2202979 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:45:46.923320', 'startedAt': '2025-01-07T22:45:46.507480', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.48325, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.893, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 776.5426712036133}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-07 17:45:46,929 INFO    HandlerThread:2202979 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-07 17:45:46,929 INFO    HandlerThread:2202979 [system_monitor.py:probe():227] Publishing system info
+2025-01-07 17:45:46,929 DEBUG   HandlerThread:2202979 [system_info.py:_save_code():44] Saving code
+2025-01-07 17:45:46,935 DEBUG   HandlerThread:2202979 [system_info.py:_save_code():67] Saving code done
+2025-01-07 17:45:46,935 DEBUG   HandlerThread:2202979 [system_info.py:_save_patches():84] Saving git patches
+2025-01-07 17:45:46,996 DEBUG   HandlerThread:2202979 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-07 17:45:46,996 INFO    HandlerThread:2202979 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-07 17:45:47,000 DEBUG   SenderThread:2202979 [sender.py:send():379] send: files
+2025-01-07 17:45:47,000 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-07 17:45:47,000 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-07 17:45:47,000 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-07 17:45:47,242 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-07 17:45:47,243 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:45:47,243 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: python_packages
+2025-01-07 17:45:47,244 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/xxe974s3-wandb-metadata.json
+2025-01-07 17:45:47,244 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:45:47,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:47,246 INFO    HandlerThread:2202979 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/tb"
+  save: true
+}
+control {
+  relay_id: "0b5c39f7df294feaa5e841e15f34f16c"
+}
+_info {
+  stream_id: "w3dz1sd1"
+}
+
+2025-01-07 17:45:47,246 DEBUG   HandlerThread:2202979 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-07 17:45:47,280 INFO    wandb-upload_1:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/04xdhoc8-code/FISH/eval_robot.py
+2025-01-07 17:45:47,292 INFO    wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/3vwzzu25-diff.patch
+2025-01-07 17:45:47,408 DEBUG   SenderThread:2202979 [sender.py:send():379] send: telemetry
+2025-01-07 17:45:47,408 DEBUG   SenderThread:2202979 [sender.py:send():379] send: tbrecord
+2025-01-07 17:45:47,409 DEBUG   SenderThread:2202979 [sender.py:send():379] send: files
+2025-01-07 17:45:47,409 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 with policy live
+2025-01-07 17:45:47,597 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/s6cmwij8-events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:45:47,846 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:45:47,846 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code/FISH/eval_robot.py
+2025-01-07 17:45:47,846 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/requirements.txt
+2025-01-07 17:45:47,846 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/diff.patch
+2025-01-07 17:45:47,846 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:45:47,846 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-metadata.json
+2025-01-07 17:45:47,847 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code/FISH
+2025-01-07 17:45:47,847 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code
+2025-01-07 17:45:48,243 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:49,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:49,845 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:45:50,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:51,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:52,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:45:52,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:53,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:54,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:55,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:56,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:57,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:58,243 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:45:58,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:45:59,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:00,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:01,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:02,243 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:46:02,243 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:46:02,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:03,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:03,321 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:46:04,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:05,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:06,246 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:07,246 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:08,246 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:09,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:46:09,246 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:10,246 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:11,246 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:12,247 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:13,067 DEBUG   SenderThread:2202979 [sender.py:send():379] send: config
+2025-01-07 17:46:13,247 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:14,247 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:15,117 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:46:15,247 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:15,852 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:16,247 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:17,243 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:46:17,244 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:46:17,247 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:17,853 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:18,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:19,247 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:19,853 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:20,247 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:20,506 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:46:20,854 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml
+2025-01-07 17:46:21,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:21,854 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:22,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:23,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:24,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:25,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:25,855 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:26,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:26,517 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:46:27,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:28,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:29,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:29,856 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:30,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:31,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:32,243 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:46:32,243 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:46:32,289 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:32,322 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:46:33,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:34,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:35,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:36,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:37,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:37,858 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:38,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:46:38,250 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:39,250 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:40,250 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:41,250 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:42,250 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:43,250 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:43,446 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:46:43,860 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:44,250 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:45,251 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:45,860 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:46,255 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:46,913 DEBUG   SystemMonitor:2202979 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-07 17:46:46,914 DEBUG   SenderThread:2202979 [sender.py:send():379] send: stats
+2025-01-07 17:46:47,243 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:46:47,243 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:46:47,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:48,252 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:49,253 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:49,351 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:46:50,252 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:51,253 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:52,253 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:53,253 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:54,253 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:54,944 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:46:55,155 DEBUG   SenderThread:2202979 [sender.py:send():379] send: files
+2025-01-07 17:46:55,155 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 with policy now
+2025-01-07 17:46:55,155 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:46:55,156 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:46:55,156 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:46:55,158 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:46:55,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:55,410 INFO    wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/zdx99p5c-media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4
+2025-01-07 17:46:55,863 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:55,863 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4
+2025-01-07 17:46:55,863 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:46:55,864 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval
+2025-01-07 17:46:55,864 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos
+2025-01-07 17:46:55,864 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media
+2025-01-07 17:46:56,179 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/70zwrw9s-media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4
+2025-01-07 17:46:56,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:57,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:57,864 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:46:58,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:59,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:46:59,865 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:47:00,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:00,909 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:01,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:02,243 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:47:02,244 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:47:02,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:03,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:03,866 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:47:04,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:05,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:06,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:06,397 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:07,255 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:08,255 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:09,255 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:10,255 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:11,255 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:12,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:12,255 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:13,255 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:14,255 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:15,256 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:16,256 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:16,914 DEBUG   SenderThread:2202979 [sender.py:send():379] send: stats
+2025-01-07 17:47:17,243 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:47:17,244 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:47:17,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:17,318 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:17,870 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:47:18,130 INFO    wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/3uotnjkj-events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:47:18,256 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:19,256 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:19,871 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:47:20,256 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:21,256 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:22,257 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:23,246 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:23,257 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:23,872 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:47:24,257 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:25,257 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:26,257 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:27,258 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:28,246 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:28,258 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:29,258 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:30,258 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:31,258 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:31,875 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:47:32,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:47:32,244 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:47:32,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:33,258 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:33,333 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:33,876 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:47:34,258 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:35,258 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:36,259 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:37,259 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:38,259 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:39,246 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:39,259 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:40,259 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:41,260 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:42,259 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:43,259 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:44,246 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:44,260 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:45,260 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:46,260 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:46,915 DEBUG   SenderThread:2202979 [sender.py:send():379] send: stats
+2025-01-07 17:47:47,244 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:47:47,244 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:47:47,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:48,260 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:49,268 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:49,350 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:49,393 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:47:49,394 DEBUG   SenderThread:2202979 [sender.py:send():379] send: files
+2025-01-07 17:47:49,394 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 with policy now
+2025-01-07 17:47:49,394 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:47:49,394 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:47:49,394 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:47:49,681 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/t5qr5l35-media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4
+2025-01-07 17:47:49,880 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:47:49,881 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:47:49,881 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4
+2025-01-07 17:47:49,881 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval
+2025-01-07 17:47:50,214 INFO    wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/vfpade1g-media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4
+2025-01-07 17:47:50,266 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:51,270 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:51,881 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:47:52,271 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:53,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:54,280 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:55,169 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:47:55,271 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:55,882 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:47:56,271 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:57,271 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:57,883 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:47:58,271 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:58,686 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:47:58,693 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:47:58,699 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:47:58,884 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:47:58,884 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:47:59,271 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:47:59,884 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:00,253 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:00,271 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:01,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:01,885 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:02,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:48:02,245 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:48:02,271 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:03,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:03,885 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:04,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:05,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:05,362 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:06,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:07,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:08,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:09,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:10,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:11,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:11,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:12,272 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:13,273 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:14,273 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:15,273 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:16,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:16,273 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:16,915 DEBUG   SenderThread:2202979 [sender.py:send():379] send: stats
+2025-01-07 17:48:17,245 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:48:17,246 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:48:17,289 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:18,281 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:19,282 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:20,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:21,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:21,352 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:22,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:23,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:23,892 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:24,293 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:25,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:26,285 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:27,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:27,286 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:28,286 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:29,296 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:29,894 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:30,147 DEBUG   SenderThread:2202979 [sender.py:send():379] send: files
+2025-01-07 17:48:30,147 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 with policy now
+2025-01-07 17:48:30,150 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:48:30,150 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:48:30,151 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:48:30,151 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:48:30,287 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:30,431 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/2xhxasnp-media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4
+2025-01-07 17:48:30,894 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:48:30,894 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4
+2025-01-07 17:48:30,894 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval
+2025-01-07 17:48:31,224 INFO    wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/2e8z4ic1-media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4
+2025-01-07 17:48:31,290 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:31,894 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:32,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:32,258 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:48:32,258 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:48:32,301 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:33,294 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:34,297 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:35,297 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:35,896 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:36,295 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:37,256 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:37,312 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:37,896 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:38,299 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:39,306 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:39,515 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:48:39,515 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:48:39,515 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:48:39,897 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:48:39,897 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:48:40,153 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/azqmx253-events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:48:40,300 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:41,311 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:41,898 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:42,264 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:42,303 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:43,330 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:44,317 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:45,317 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:45,899 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:46,317 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:46,916 DEBUG   SenderThread:2202979 [sender.py:send():379] send: stats
+2025-01-07 17:48:47,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:48:47,248 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:48:47,317 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:47,344 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:48,317 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:49,317 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:50,318 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:51,319 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:51,901 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:52,319 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:53,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:53,319 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:54,319 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:55,319 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:55,903 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:56,319 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:57,319 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:57,903 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:48:58,320 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:48:59,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:48:59,320 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:00,320 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:01,320 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:02,248 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:49:02,248 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:49:02,320 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:03,320 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:04,320 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:04,385 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:05,320 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:06,321 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:07,321 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:08,321 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:09,321 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:10,242 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:10,321 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:11,333 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:11,908 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:49:12,131 DEBUG   SenderThread:2202979 [sender.py:send():379] send: files
+2025-01-07 17:49:12,131 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 with policy now
+2025-01-07 17:49:12,135 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:49:12,136 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:49:12,136 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:49:12,136 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:49:12,325 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:12,406 INFO    wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/yfj4yw0y-media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4
+2025-01-07 17:49:12,908 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:49:12,909 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4
+2025-01-07 17:49:12,909 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval
+2025-01-07 17:49:13,247 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/x02skg7i-media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4
+2025-01-07 17:49:13,342 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:13,909 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:49:14,332 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:14,909 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:49:15,254 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:15,332 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:16,336 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:16,916 DEBUG   SenderThread:2202979 [sender.py:send():379] send: stats
+2025-01-07 17:49:17,252 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:49:17,252 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:49:17,338 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:17,910 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:49:18,338 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:19,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:19,911 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:49:20,336 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:20,725 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:21,337 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:21,912 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:49:22,341 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:22,568 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:49:22,568 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:49:22,569 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:49:22,912 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:49:23,339 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:23,912 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:49:24,342 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:25,344 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:25,741 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:26,339 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:27,339 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:27,914 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:49:28,339 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:29,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:30,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:31,259 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:31,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:32,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:49:32,249 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:49:32,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:33,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:34,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:35,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:36,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:36,407 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:37,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:38,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:39,340 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:40,341 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:41,341 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:42,256 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:42,341 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:43,341 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:44,341 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:45,341 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:46,342 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:46,917 DEBUG   SenderThread:2202979 [sender.py:send():379] send: stats
+2025-01-07 17:49:47,249 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:49:47,250 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:49:47,342 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:47,353 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:48,342 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:49,342 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:50,342 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:51,342 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:52,342 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:53,044 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:53,357 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:53,924 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:49:54,345 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:54,513 DEBUG   SenderThread:2202979 [sender.py:send():379] send: files
+2025-01-07 17:49:54,513 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 with policy now
+2025-01-07 17:49:54,520 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:49:54,521 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:49:54,521 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:49:54,521 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:49:54,789 INFO    wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/8qa6tahn-media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4
+2025-01-07 17:49:54,925 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:49:54,925 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4
+2025-01-07 17:49:54,925 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval
+2025-01-07 17:49:55,253 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/klnldrkk-media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4
+2025-01-07 17:49:55,350 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:55,925 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:49:56,364 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:56,925 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:49:57,363 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:58,274 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:49:58,375 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:59,364 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:49:59,926 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:50:00,363 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:01,378 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:01,927 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:50:02,261 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:50:02,261 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:50:02,366 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:03,380 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:03,418 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:50:03,928 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:50:04,366 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:05,369 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:06,390 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:07,380 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:08,388 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:09,284 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:50:09,387 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:10,393 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:11,391 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:12,393 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:12,635 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:50:12,635 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:12,636 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:12,803 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 17:50:12,804 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 17:50:12,931 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:50:12,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:50:12,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:50:12,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media
+2025-01-07 17:50:12,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table
+2025-01-07 17:50:12,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval
+2025-01-07 17:50:13,066 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:50:13,132 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/kooouc71-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:50:13,216 INFO    wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/76qc2sny-events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:50:13,369 INFO    wandb-upload_1:2202979 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0s0ujrp5
+2025-01-07 17:50:13,390 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:13,445 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: log_artifact
+2025-01-07 17:50:13,489 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-07 17:50:13,744 INFO    SenderThread:2202979 [sender.py:send_request_log_artifact():1525] logged artifact run-w3dz1sd1-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAwNDY5MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcyMzIzOA==', 'latestArtifact': None}}
+2025-01-07 17:50:13,744 DEBUG   SenderThread:2202979 [sender.py:send():379] send: files
+2025-01-07 17:50:13,744 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now
+2025-01-07 17:50:13,744 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:50:13,744 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,745 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,745 DEBUG   SenderThread:2202979 [sender.py:send():379] send: telemetry
+2025-01-07 17:50:13,745 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,749 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,749 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,749 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,749 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,750 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,750 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,750 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,751 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,751 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,751 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,752 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,752 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,752 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,752 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,753 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,753 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,753 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,753 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,754 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,754 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:13,754 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:13,754 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: log_artifact
+2025-01-07 17:50:13,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:50:13,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:50:13,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:50:13,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json
+2025-01-07 17:50:13,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval
+2025-01-07 17:50:14,007 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/e994hzpj-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:50:14,167 INFO    wandb-upload_1:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/fathqyru-media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json
+2025-01-07 17:50:14,193 INFO    wandb-upload_2:2202979 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjmevjhc8
+2025-01-07 17:50:14,390 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:14,580 INFO    SenderThread:2202979 [sender.py:send_request_log_artifact():1525] logged artifact run-w3dz1sd1-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjAwNDc5NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcyMzMxMw==', 'latestArtifact': None}}
+2025-01-07 17:50:14,580 DEBUG   SenderThread:2202979 [sender.py:send():379] send: files
+2025-01-07 17:50:14,580 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json with policy now
+2025-01-07 17:50:14,580 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:50:14,581 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:50:14,581 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:14,581 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:14,839 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/yg18e0j6-media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json
+2025-01-07 17:50:14,932 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:50:15,391 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:15,933 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:50:16,391 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:16,917 DEBUG   SenderThread:2202979 [sender.py:send():379] send: stats
+2025-01-07 17:50:17,251 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-07 17:50:17,251 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: stop_status
+2025-01-07 17:50:17,391 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:17,809 DEBUG   SenderThread:2202979 [sender.py:send():379] send: exit
+2025-01-07 17:50:17,809 INFO    SenderThread:2202979 [sender.py:send_exit():586] handling exit code: 0
+2025-01-07 17:50:17,810 INFO    SenderThread:2202979 [sender.py:send_exit():588] handling runtime: 270
+2025-01-07 17:50:17,810 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:17,810 INFO    SenderThread:2202979 [sender.py:send_exit():594] send defer
+2025-01-07 17:50:17,810 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:17,810 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-07 17:50:17,810 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:17,810 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-07 17:50:17,810 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 1
+2025-01-07 17:50:17,810 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:17,811 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-07 17:50:17,811 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:17,811 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-07 17:50:17,811 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 2
+2025-01-07 17:50:17,811 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:17,811 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-07 17:50:17,811 INFO    HandlerThread:2202979 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-07 17:50:17,811 INFO    HandlerThread:2202979 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-07 17:50:17,811 DEBUG   SystemMonitor:2202979 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-07 17:50:17,811 DEBUG   SystemMonitor:2202979 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-07 17:50:17,812 INFO    HandlerThread:2202979 [interfaces.py:finish():200] Joined disk monitor
+2025-01-07 17:50:17,889 INFO    HandlerThread:2202979 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-07 17:50:17,889 INFO    HandlerThread:2202979 [interfaces.py:finish():200] Joined memory monitor
+2025-01-07 17:50:17,889 INFO    HandlerThread:2202979 [interfaces.py:finish():200] Joined network monitor
+2025-01-07 17:50:17,890 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:17,890 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-07 17:50:17,890 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 3
+2025-01-07 17:50:17,890 DEBUG   SenderThread:2202979 [sender.py:send():379] send: stats
+2025-01-07 17:50:17,890 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:17,890 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-07 17:50:17,890 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:17,890 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-07 17:50:17,890 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 4
+2025-01-07 17:50:17,891 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:17,891 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-07 17:50:17,934 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:50:19,934 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:50:24,641 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:50:24,641 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:50:24,641 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-07 17:50:24,643 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:24,643 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-07 17:50:24,643 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 5
+2025-01-07 17:50:24,643 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:50:24,643 DEBUG   SenderThread:2202979 [sender.py:send():379] send: history
+2025-01-07 17:50:24,643 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:24,643 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: summary_record
+2025-01-07 17:50:24,644 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-07 17:50:24,645 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:24,645 DEBUG   SenderThread:2202979 [sender.py:send():379] send: summary
+2025-01-07 17:50:24,649 INFO    SenderThread:2202979 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-07 17:50:24,649 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:24,649 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-07 17:50:24,649 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 6
+2025-01-07 17:50:24,649 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:24,649 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-07 17:50:24,650 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:24,650 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-07 17:50:24,653 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: status_report
+2025-01-07 17:50:24,774 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 7
+2025-01-07 17:50:24,774 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:24,774 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-07 17:50:24,775 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:24,775 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-07 17:50:24,811 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:50:24,936 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml
+2025-01-07 17:50:24,937 INFO    Thread-12 :2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:50:26,055 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 8
+2025-01-07 17:50:26,055 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:50:26,055 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:26,055 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-07 17:50:26,055 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:26,055 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-07 17:50:26,055 INFO    SenderThread:2202979 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-07 17:50:26,058 INFO    SenderThread:2202979 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-07 17:50:26,060 INFO    SenderThread:2202979 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-07 17:50:26,072 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 9
+2025-01-07 17:50:26,072 DEBUG   SenderThread:2202979 [sender.py:send():379] send: artifact
+2025-01-07 17:50:26,075 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:26,075 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-07 17:50:26,300 INFO    SenderThread:2202979 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg5NjA2OQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}}
+2025-01-07 17:50:26,301 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:26,301 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-07 17:50:26,301 INFO    SenderThread:2202979 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-07 17:50:26,811 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:50:26,938 INFO    SenderThread:2202979 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:50:26,938 INFO    SenderThread:2202979 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files
+2025-01-07 17:50:26,938 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml config.yaml
+2025-01-07 17:50:26,938 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json wandb-summary.json
+2025-01-07 17:50:26,938 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0 events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:50:26,938 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/requirements.txt requirements.txt
+2025-01-07 17:50:26,939 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-metadata.json wandb-metadata.json
+2025-01-07 17:50:26,939 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log output.log
+2025-01-07 17:50:26,939 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/diff.patch diff.patch
+2025-01-07 17:50:26,943 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-07 17:50:26,943 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4 media/videos/eval/3_eval_3_ea44f1493fcbb8437821.mp4
+2025-01-07 17:50:26,943 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4 media/videos/eval/0_eval_0_6e4ad6e0af4415805224.mp4
+2025-01-07 17:50:26,943 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4 media/videos/eval/4_eval_4_cadcc71b7662da48835a.mp4
+2025-01-07 17:50:26,943 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4 media/videos/eval/1_eval_1_f499daa5972d95524f38.mp4
+2025-01-07 17:50:26,943 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4 media/videos/eval/2_eval_2_77080ebc0cccad9e8ca8.mp4
+2025-01-07 17:50:26,943 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json
+2025-01-07 17:50:26,943 INFO    SenderThread:2202979 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json media/table/eval/total_success_rate_ci_6_fedf8048f40139f691f9.table.json
+2025-01-07 17:50:26,943 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 10
+2025-01-07 17:50:26,943 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:50:26,944 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:26,944 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-07 17:50:26,944 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:26,944 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-07 17:50:26,944 INFO    SenderThread:2202979 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 17:50:27,133 INFO    wandb-upload_1:2202979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/wandb-summary.json
+2025-01-07 17:50:27,166 INFO    wandb-upload_2:2202979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/config.yaml
+2025-01-07 17:50:27,174 INFO    wandb-upload_0:2202979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/output.log
+2025-01-07 17:50:27,196 INFO    wandb-upload_3:2202979 [upload_job.py:push():130] Uploaded file /tmp/tmpxg5s52pnwandb/szi5z8ek-events.out.tfevents.1736289947.leonmkim-ROG-Strix-G15CS-G15CS.2202791.0
+2025-01-07 17:50:27,199 INFO    wandb-upload_4:2202979 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/files/requirements.txt
+2025-01-07 17:50:27,399 INFO    Thread-11 (_thread_body):2202979 [sender.py:transition_state():614] send defer: 11
+2025-01-07 17:50:27,399 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:27,399 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-07 17:50:27,399 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:27,399 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-07 17:50:27,400 INFO    SenderThread:2202979 [file_pusher.py:join():175] waiting for file pusher
+2025-01-07 17:50:27,401 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 12
+2025-01-07 17:50:27,401 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:27,401 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-07 17:50:27,401 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:27,401 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-07 17:50:27,401 INFO    SenderThread:2202979 [file_stream.py:finish():601] file stream finish called
+2025-01-07 17:50:27,613 INFO    SenderThread:2202979 [file_stream.py:finish():605] file stream finish is done
+2025-01-07 17:50:27,613 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 13
+2025-01-07 17:50:27,613 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:27,613 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-07 17:50:27,613 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:27,613 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-07 17:50:27,613 INFO    SenderThread:2202979 [sender.py:transition_state():614] send defer: 14
+2025-01-07 17:50:27,613 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: defer
+2025-01-07 17:50:27,613 INFO    HandlerThread:2202979 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-07 17:50:27,614 DEBUG   SenderThread:2202979 [sender.py:send():379] send: final
+2025-01-07 17:50:27,614 DEBUG   SenderThread:2202979 [sender.py:send():379] send: footer
+2025-01-07 17:50:27,614 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: defer
+2025-01-07 17:50:27,614 INFO    SenderThread:2202979 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-07 17:50:27,614 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:50:27,614 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:50:27,615 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: server_info
+2025-01-07 17:50:27,615 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-07 17:50:27,615 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-07 17:50:27,616 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-07 17:50:27,616 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: server_info
+2025-01-07 17:50:27,616 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-07 17:50:27,678 DEBUG   SenderThread:2202979 [sender.py:send_request():406] send_request: poll_exit
+2025-01-07 17:50:27,679 INFO    MainThread:2202979 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-07 17:50:27,679 INFO    MainThread:2202979 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-07 17:50:27,679 INFO    MainThread:2202979 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2025-01-07 17:50:27,679 DEBUG   HandlerThread:2202979 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-07 17:50:27,679 INFO    HandlerThread:2202979 [handler.py:finish():882] shutting down handler
+2025-01-07 17:50:28,615 INFO    WriterThread:2202979 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/run-w3dz1sd1.wandb
+2025-01-07 17:50:28,678 INFO    SenderThread:2202979 [sender.py:finish():1615] shutting down sender
+2025-01-07 17:50:28,679 INFO    SenderThread:2202979 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-07 17:50:28,679 INFO    SenderThread:2202979 [file_pusher.py:join():175] waiting for file pusher
diff --git a/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/logs/debug.log b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/logs/debug.log
new file mode 100644
index 0000000000000000000000000000000000000000..0b7f672983b5dbe9dfa7f4d58d55799a31655ffa
--- /dev/null
+++ b/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/logs/debug.log
@@ -0,0 +1,31 @@
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_setup.py:_flush():76] Configure stats pid to 2202791
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/settings
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/logs/debug.log
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/wandb/run-20250107_174546-w3dz1sd1/logs/debug-internal.log
+2025-01-07 17:45:46,514 INFO    MainThread:2202791 [wandb_init.py:init():569] calling init triggers
+2025-01-07 17:45:46,515 INFO    MainThread:2202791 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1009_1', 'wandb_notes': '1009_1_restarted_3', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1'}
+2025-01-07 17:45:46,515 INFO    MainThread:2202791 [wandb_init.py:init():619] starting backend
+2025-01-07 17:45:46,515 INFO    MainThread:2202791 [wandb_init.py:init():623] setting up manager
+2025-01-07 17:45:46,519 INFO    MainThread:2202791 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-07 17:45:46,520 INFO    MainThread:2202791 [wandb_init.py:init():631] backend started and connected
+2025-01-07 17:45:46,532 INFO    MainThread:2202791 [wandb_init.py:init():720] updated telemetry
+2025-01-07 17:45:46,539 INFO    MainThread:2202791 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-07 17:45:46,859 INFO    MainThread:2202791 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-07 17:45:46,900 INFO    MainThread:2202791 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-07 17:45:46,900 INFO    MainThread:2202791 [wandb_init.py:init():804] starting run threads in backend
+2025-01-07 17:45:47,243 INFO    MainThread:2202791 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-07 17:45:47,243 INFO    MainThread:2202791 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-07 17:45:47,243 INFO    MainThread:2202791 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-07 17:45:47,243 INFO    MainThread:2202791 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-07 17:45:47,245 INFO    MainThread:2202791 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-07 17:45:47,245 INFO    MainThread:2202791 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/w3dz1sd1/tb, True
+2025-01-07 17:46:13,067 INFO    MainThread:2202791 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'}
+2025-01-07 17:50:28,680 WARNING MsgRouterThr:2202791 [router.py:message_loop():77] message_loop has been closed
diff --git a/xu8zuth0/.hydra/config.yaml b/xu8zuth0/.hydra/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5c9f448d7baed90316fd960de3f5cad4cbc9c2ab
--- /dev/null
+++ b/xu8zuth0/.hydra/config.yaml
@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1009_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
diff --git a/xu8zuth0/.hydra/hydra.yaml b/xu8zuth0/.hydra/hydra.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ea0d3e6db2614de8cc0fdfa9c8ee8ec309752203
--- /dev/null
+++ b/xu8zuth0/.hydra/hydra.yaml
@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1009_1/xu8zuth0
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false
diff --git a/xu8zuth0/.hydra/overrides.yaml b/xu8zuth0/.hydra/overrides.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc
--- /dev/null
+++ b/xu8zuth0/.hydra/overrides.yaml
@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion
diff --git a/xu8zuth0/eval_policy.log b/xu8zuth0/eval_policy.log
new file mode 100644
index 0000000000000000000000000000000000000000..e3b69f02286d113856e012d923b7ee3a442ea876
--- /dev/null
+++ b/xu8zuth0/eval_policy.log
@@ -0,0 +1,15 @@
+[2025-01-06 22:08:57,734][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-06 22:08:57,737][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-06 22:09:01,136][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+