Add files using upload-large-folder tool

.gitattributes

@@ -94,3 +94,49 @@ pzxh5op7/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
 nl3lmlc3/episode_rosbags/episode_4_2025-01-04-21-37-56.bag filter=lfs diff=lfs merge=lfs -text
 nl3lmlc3/episode_rosbags/episode_1_2025-01-04-21-35-23.bag filter=lfs diff=lfs merge=lfs -text
 y2m69p6z/episode_rosbags/episode_3_2025-01-04-20-46-22.bag filter=lfs diff=lfs merge=lfs -text
+zc0axaef/episode_rosbags/episode_1_2025-01-04-21-30-17.bag filter=lfs diff=lfs merge=lfs -text
+zc0axaef/episode_rosbags/episode_2_2025-01-04-21-31-11.bag filter=lfs diff=lfs merge=lfs -text
+zc0axaef/episode_rosbags/episode_0_2025-01-04-21-29-24.bag filter=lfs diff=lfs merge=lfs -text
+zc0axaef/episode_rosbags/episode_3_2025-01-04-21-32-04.bag filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/0_eval_0_16d80c9a1dc6763cf29a.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/2_eval_2_4982ced87e18fd534ea7.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/4_eval_4_06b29357871c19fff932.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/3_eval_3_e8c539eab0fc6faa4df8.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/wandb/run-20250104_213413-nl3lmlc3/files/media/videos/eval/1_eval_1_c9d9568f97ca21d8f63a.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/episode_rosbags/episode_2_2025-01-04-21-36-11.bag filter=lfs diff=lfs merge=lfs -text
+nayavtp6/episode_rosbags/episode_1_2025-01-03-21-17-13.bag filter=lfs diff=lfs merge=lfs -text
+nayavtp6/episode_rosbags/episode_2_2025-01-03-21-17-55.bag filter=lfs diff=lfs merge=lfs -text
+zc0axaef/episode_rosbags/episode_4_2025-01-04-21-32-54.bag filter=lfs diff=lfs merge=lfs -text
+nayavtp6/episode_rosbags/episode_0_2025-01-03-21-16-31.bag filter=lfs diff=lfs merge=lfs -text
+nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/1_eval_1_91c0aab1f5aa1b260cf6.mp4 filter=lfs diff=lfs merge=lfs -text
+nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/3_eval_3_a145a501509e034700f1.mp4 filter=lfs diff=lfs merge=lfs -text
+nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/4_eval_4_8ec81cfff6351c2b5db4.mp4 filter=lfs diff=lfs merge=lfs -text
+nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/2_eval_2_a653d18806cfc6db0e07.mp4 filter=lfs diff=lfs merge=lfs -text
+nayavtp6/wandb/run-20250103_211606-nayavtp6/files/media/videos/eval/0_eval_0_29e1a738e40422f19100.mp4 filter=lfs diff=lfs merge=lfs -text
+nayavtp6/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nayavtp6/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nayavtp6/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nayavtp6/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nayavtp6/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+p96n5ha2/episode_rosbags/episode_1_2025-01-03-22-59-41.bag filter=lfs diff=lfs merge=lfs -text
+nayavtp6/episode_rosbags/episode_4_2025-01-03-21-19-19.bag filter=lfs diff=lfs merge=lfs -text
+iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/4_eval_4_d691d8c6f8564711e47c.mp4 filter=lfs diff=lfs merge=lfs -text
+nayavtp6/episode_rosbags/episode_3_2025-01-03-21-18-32.bag filter=lfs diff=lfs merge=lfs -text
+iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/0_eval_0_3f829840c9d5231f7c2a.mp4 filter=lfs diff=lfs merge=lfs -text
+iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/2_eval_2_fef06f9228f28f137565.mp4 filter=lfs diff=lfs merge=lfs -text
+iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/1_eval_1_fcc2c84034b2f80493e2.mp4 filter=lfs diff=lfs merge=lfs -text
+iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/videos/eval/3_eval_3_2ceb841f3e55434b9bc7.mp4 filter=lfs diff=lfs merge=lfs -text
+iqjkgl2e/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+iqjkgl2e/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+iqjkgl2e/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+iqjkgl2e/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+iqjkgl2e/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/episode_rosbags/episode_0_2025-01-04-21-34-30.bag filter=lfs diff=lfs merge=lfs -text
+nl3lmlc3/episode_rosbags/episode_3_2025-01-04-21-37-04.bag filter=lfs diff=lfs merge=lfs -text
+48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 filter=lfs diff=lfs merge=lfs -text
+p96n5ha2/episode_rosbags/episode_3_2025-01-03-23-01-30.bag filter=lfs diff=lfs merge=lfs -text

48t61yqi/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1926_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

48t61yqi/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

48t61yqi/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

48t61yqi/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

48t61yqi/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

48t61yqi/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

48t61yqi/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

48t61yqi/eval_robot.log

@@ -0,0 +1,12 @@
+[2025-01-03 21:13:01,810][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-03 21:13:01,813][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

48t61yqi/tb/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:460d0b1754c3dff8072e67e31161b3fcda23ca39c7a25f6b6ffc5c5d26f1400f
+size 1171

48t61yqi/wandb/debug-internal.log

@@ -0,0 +1,468 @@
+2025-01-03 21:13:07,574 INFO    StreamThr :1974795 [internal.py:wandb_internal():85] W&B internal server running at pid: 1974795, started at: 2025-01-03 21:13:07.573731
+2025-01-03 21:13:07,576 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status
+2025-01-03 21:13:07,587 INFO    WriterThread:1974795 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb
+2025-01-03 21:13:07,587 DEBUG   SenderThread:1974795 [sender.py:send():379] send: header
+2025-01-03 21:13:07,600 DEBUG   SenderThread:1974795 [sender.py:send():379] send: run
+2025-01-03 21:13:07,945 INFO    SenderThread:1974795 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files
+2025-01-03 21:13:07,945 INFO    SenderThread:1974795 [sender.py:_start_run_threads():1188] run started: 48t61yqi with start time 1735956787.574555
+2025-01-03 21:13:07,961 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: check_version
+2025-01-03 21:13:07,961 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: check_version
+2025-01-03 21:13:08,006 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: run_start
+2025-01-03 21:13:08,011 DEBUG   HandlerThread:1974795 [system_info.py:__init__():26] System info init
+2025-01-03 21:13:08,011 DEBUG   HandlerThread:1974795 [system_info.py:__init__():41] System info init done
+2025-01-03 21:13:08,011 INFO    HandlerThread:1974795 [system_monitor.py:start():194] Starting system monitor
+2025-01-03 21:13:08,011 INFO    SystemMonitor:1974795 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-03 21:13:08,011 INFO    HandlerThread:1974795 [system_monitor.py:probe():214] Collecting system info
+2025-01-03 21:13:08,012 INFO    SystemMonitor:1974795 [interfaces.py:start():188] Started cpu monitoring
+2025-01-03 21:13:08,012 INFO    SystemMonitor:1974795 [interfaces.py:start():188] Started disk monitoring
+2025-01-03 21:13:08,013 INFO    SystemMonitor:1974795 [interfaces.py:start():188] Started gpu monitoring
+2025-01-03 21:13:08,013 INFO    SystemMonitor:1974795 [interfaces.py:start():188] Started memory monitoring
+2025-01-03 21:13:08,013 INFO    SystemMonitor:1974795 [interfaces.py:start():188] Started network monitoring
+2025-01-03 21:13:08,025 DEBUG   HandlerThread:1974795 [system_info.py:probe():152] Probing system
+2025-01-03 21:13:08,027 DEBUG   HandlerThread:1974795 [system_info.py:_probe_git():137] Probing git
+2025-01-03 21:13:08,031 DEBUG   HandlerThread:1974795 [system_info.py:_probe_git():145] Probing git done
+2025-01-03 21:13:08,031 DEBUG   HandlerThread:1974795 [system_info.py:probe():200] Probing system done
+2025-01-03 21:13:08,031 DEBUG   HandlerThread:1974795 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T02:13:08.025575', 'startedAt': '2025-01-04T02:13:07.566812', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.002625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.061, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 804.7593460083008}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-03 21:13:08,031 INFO    HandlerThread:1974795 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-03 21:13:08,031 INFO    HandlerThread:1974795 [system_monitor.py:probe():227] Publishing system info
+2025-01-03 21:13:08,031 DEBUG   HandlerThread:1974795 [system_info.py:_save_code():44] Saving code
+2025-01-03 21:13:08,033 DEBUG   HandlerThread:1974795 [system_info.py:_save_code():67] Saving code done
+2025-01-03 21:13:08,033 DEBUG   HandlerThread:1974795 [system_info.py:_save_patches():84] Saving git patches
+2025-01-03 21:13:08,094 DEBUG   HandlerThread:1974795 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-03 21:13:08,095 INFO    HandlerThread:1974795 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-03 21:13:08,098 DEBUG   SenderThread:1974795 [sender.py:send():379] send: files
+2025-01-03 21:13:08,098 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-03 21:13:08,098 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-03 21:13:08,098 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-03 21:13:08,340 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/7zaf7m84-diff.patch
+2025-01-03 21:13:08,350 INFO    wandb-upload_0:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/2v3neepq-wandb-metadata.json
+2025-01-03 21:13:08,359 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-03 21:13:08,360 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: python_packages
+2025-01-03 21:13:08,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:13:08,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:08,361 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:13:08,362 INFO    HandlerThread:1974795 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/tb"
+  save: true
+}
+control {
+  relay_id: "106e59ccfedd40bd8337b716b182d46a"
+}
+_info {
+  stream_id: "48t61yqi"
+}
+
+2025-01-03 21:13:08,362 DEBUG   HandlerThread:1974795 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-03 21:13:08,373 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/wfd4dtnr-code/FISH/eval_robot.py
+2025-01-03 21:13:08,515 DEBUG   SenderThread:1974795 [sender.py:send():379] send: telemetry
+2025-01-03 21:13:08,515 DEBUG   SenderThread:1974795 [sender.py:send():379] send: tbrecord
+2025-01-03 21:13:08,515 DEBUG   SenderThread:1974795 [sender.py:send():379] send: files
+2025-01-03 21:13:08,515 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 with policy live
+2025-01-03 21:13:08,698 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/utluf6qk-events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0
+2025-01-03 21:13:08,946 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt
+2025-01-03 21:13:08,946 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code
+2025-01-03 21:13:09,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:10,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:10,945 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:11,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:12,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:13,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:13,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:14,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:15,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:16,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:17,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:18,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:18,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:19,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:20,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:21,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:22,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:23,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:13:23,360 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:13:23,401 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:23,464 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:24,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:25,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:25,966 DEBUG   SenderThread:1974795 [sender.py:send():379] send: config
+2025-01-03 21:13:26,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:26,950 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:27,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:28,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:28,950 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:29,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:29,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:30,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:30,951 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:31,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:32,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:32,951 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:33,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:34,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:34,461 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:34,952 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:35,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:36,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:37,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:38,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:13:38,360 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:13:38,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:38,953 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:39,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:39,520 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:39,953 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml
+2025-01-03 21:13:40,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:41,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:42,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:42,954 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:43,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:44,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:45,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:45,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:46,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:47,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:48,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:49,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:50,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:50,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:51,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:52,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:53,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:13:53,360 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:13:53,401 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:54,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:55,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:55,497 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:56,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:57,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:58,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:59,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:00,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:01,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:01,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:02,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:03,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:04,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:05,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:06,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:07,313 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:07,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:08,014 DEBUG   SystemMonitor:1974795 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-03 21:14:08,014 DEBUG   SenderThread:1974795 [sender.py:send():379] send: stats
+2025-01-03 21:14:08,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:14:08,361 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:14:08,409 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:08,961 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:09,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:10,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:11,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:12,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:12,962 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:13,073 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:13,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:14,372 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:15,370 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:16,371 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:16,652 DEBUG   SenderThread:1974795 [sender.py:send():379] send: files
+2025-01-03 21:14:16,652 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 with policy now
+2025-01-03 21:14:16,652 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-03 21:14:16,653 DEBUG   SenderThread:1974795 [sender.py:send():379] send: history
+2025-01-03 21:14:16,654 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record
+2025-01-03 21:14:16,654 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:14:16,878 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/kh7pplhk-media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4
+2025-01-03 21:14:16,963 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:16,963 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4
+2025-01-03 21:14:16,964 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json
+2025-01-03 21:14:16,964 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos
+2025-01-03 21:14:16,964 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media
+2025-01-03 21:14:16,964 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval
+2025-01-03 21:14:17,232 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/rsn814pa-media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4
+2025-01-03 21:14:17,371 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:18,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:18,372 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:18,964 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:19,373 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:20,409 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:21,372 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:22,374 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:22,965 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:23,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:14:23,361 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:14:23,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:23,479 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:24,374 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:24,966 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:25,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:26,374 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:27,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:28,374 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:29,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:29,374 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:30,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:31,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:32,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:33,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:34,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:35,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:35,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:36,376 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:37,376 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:38,015 DEBUG   SenderThread:1974795 [sender.py:send():379] send: stats
+2025-01-03 21:14:38,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:14:38,361 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:14:38,405 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:39,376 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:40,376 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:40,473 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:41,377 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:42,377 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:43,377 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:44,378 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:45,377 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:46,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:46,377 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:47,378 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:48,378 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:49,378 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:50,379 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:51,379 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:52,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:52,379 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:53,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:14:53,362 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:14:53,409 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:54,379 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:55,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:56,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:56,976 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:56,976 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0
+2025-01-03 21:14:57,223 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/2qynfcuc-events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0
+2025-01-03 21:14:57,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:58,181 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:58,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:58,976 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:59,381 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:00,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:00,977 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:01,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:01,533 DEBUG   SenderThread:1974795 [sender.py:send():379] send: files
+2025-01-03 21:15:01,533 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 with policy now
+2025-01-03 21:15:01,533 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-03 21:15:01,534 DEBUG   SenderThread:1974795 [sender.py:send():379] send: history
+2025-01-03 21:15:01,534 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record
+2025-01-03 21:15:01,535 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:15:01,978 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json
+2025-01-03 21:15:01,978 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4
+2025-01-03 21:15:01,978 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval
+2025-01-03 21:15:02,264 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/ob86mf5s-media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4
+2025-01-03 21:15:02,389 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:02,513 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/c7fg0d8r-media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4
+2025-01-03 21:15:02,978 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:03,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:03,389 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:04,384 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:04,979 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:05,383 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:06,385 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:06,979 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:07,384 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:08,015 DEBUG   SenderThread:1974795 [sender.py:send():379] send: stats
+2025-01-03 21:15:08,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:15:08,363 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:15:08,405 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:08,465 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:09,388 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:10,389 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:11,388 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:12,390 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:13,393 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:14,369 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:14,390 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:15,393 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:16,391 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:17,396 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:18,396 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:19,372 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:19,395 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:20,395 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:21,395 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:22,395 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:23,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:15:23,362 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:15:23,409 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:24,397 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:24,467 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:25,399 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:26,399 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:27,399 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:28,401 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:29,403 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:29,420 DEBUG   SenderThread:1974795 [sender.py:send():379] send: files
+2025-01-03 21:15:29,420 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 with policy now
+2025-01-03 21:15:29,465 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-03 21:15:29,465 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: server_info
+2025-01-03 21:15:29,466 DEBUG   SenderThread:1974795 [sender.py:send():379] send: history
+2025-01-03 21:15:29,466 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record
+2025-01-03 21:15:29,467 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:15:29,468 DEBUG   SenderThread:1974795 [sender.py:send():379] send: telemetry
+2025-01-03 21:15:29,468 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: server_info
+2025-01-03 21:15:29,468 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:29,537 DEBUG   SenderThread:1974795 [sender.py:send():379] send: exit
+2025-01-03 21:15:29,537 INFO    SenderThread:1974795 [sender.py:send_exit():586] handling exit code: 0
+2025-01-03 21:15:29,537 INFO    SenderThread:1974795 [sender.py:send_exit():588] handling runtime: 141
+2025-01-03 21:15:29,537 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:15:29,537 INFO    SenderThread:1974795 [sender.py:send_exit():594] send defer
+2025-01-03 21:15:29,537 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:29,537 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-03 21:15:29,538 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:29,538 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-03 21:15:29,538 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 1
+2025-01-03 21:15:29,538 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:29,538 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-03 21:15:29,538 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:29,538 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-03 21:15:29,538 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 2
+2025-01-03 21:15:29,538 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:29,538 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-03 21:15:29,538 INFO    HandlerThread:1974795 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-03 21:15:29,538 DEBUG   SystemMonitor:1974795 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-03 21:15:29,538 INFO    HandlerThread:1974795 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-03 21:15:29,538 DEBUG   SystemMonitor:1974795 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-03 21:15:29,539 INFO    HandlerThread:1974795 [interfaces.py:finish():200] Joined disk monitor
+2025-01-03 21:15:29,590 INFO    HandlerThread:1974795 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-03 21:15:29,591 INFO    HandlerThread:1974795 [interfaces.py:finish():200] Joined memory monitor
+2025-01-03 21:15:29,591 INFO    HandlerThread:1974795 [interfaces.py:finish():200] Joined network monitor
+2025-01-03 21:15:29,591 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:29,591 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-03 21:15:29,591 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 3
+2025-01-03 21:15:29,591 DEBUG   SenderThread:1974795 [sender.py:send():379] send: stats
+2025-01-03 21:15:29,591 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:29,591 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-03 21:15:29,592 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:29,592 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-03 21:15:29,592 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 4
+2025-01-03 21:15:29,592 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:29,592 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-03 21:15:29,596 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/44ln0e3q-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-03 21:15:29,987 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json
+2025-01-03 21:15:29,987 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-03 21:15:29,987 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval
+2025-01-03 21:15:30,242 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/vh0f04q3-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-03 21:15:30,987 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:36,759 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-03 21:15:36,760 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:36,760 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-03 21:15:36,760 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:36,761 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-03 21:15:36,761 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 5
+2025-01-03 21:15:36,761 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit
+2025-01-03 21:15:36,761 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:36,761 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-03 21:15:36,762 DEBUG   SenderThread:1974795 [sender.py:send():379] send: history
+2025-01-03 21:15:36,762 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record
+2025-01-03 21:15:36,762 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:15:36,762 DEBUG   SenderThread:1974795 [sender.py:send():379] send: summary
+2025-01-03 21:15:36,766 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:15:36,767 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:36,767 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-03 21:15:36,767 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 6
+2025-01-03 21:15:36,767 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:36,767 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-03 21:15:36,767 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:36,767 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-03 21:15:36,770 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:36,885 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 7
+2025-01-03 21:15:36,885 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:36,885 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-03 21:15:36,885 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:36,885 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-03 21:15:36,989 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml
+2025-01-03 21:15:36,989 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json
+2025-01-03 21:15:37,544 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-03 21:15:38,663 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 8
+2025-01-03 21:15:38,664 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit
+2025-01-03 21:15:38,664 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:38,664 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-03 21:15:38,664 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:38,664 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-03 21:15:38,664 INFO    SenderThread:1974795 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-03 21:15:38,666 INFO    SenderThread:1974795 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-03 21:15:38,680 INFO    SenderThread:1974795 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-03 21:15:38,694 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 9
+2025-01-03 21:15:38,694 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:38,694 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-03 21:15:38,694 DEBUG   SenderThread:1974795 [sender.py:send():379] send: artifact
+2025-01-03 21:15:38,990 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:39,322 INFO    wandb-upload_0:1974795 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjy2pywco
+2025-01-03 21:15:39,428 INFO    wandb-upload_2:1974795 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqkmuwojg
+2025-01-03 21:15:39,482 INFO    wandb-upload_1:1974795 [upload_job.py:push():88] Uploaded file /tmp/tmpwr759ffl/wandb-job.json
+2025-01-03 21:15:39,558 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-03 21:15:40,071 INFO    SenderThread:1974795 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg5OTM1NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'versionIndex': 408}}}
+2025-01-03 21:15:40,072 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:40,072 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-03 21:15:40,072 INFO    SenderThread:1974795 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-03 21:15:40,991 INFO    SenderThread:1974795 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files
+2025-01-03 21:15:40,991 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml config.yaml
+2025-01-03 21:15:40,991 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json wandb-summary.json
+2025-01-03 21:15:40,991 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt requirements.txt
+2025-01-03 21:15:40,991 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json wandb-metadata.json
+2025-01-03 21:15:40,993 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0
+2025-01-03 21:15:40,996 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log output.log
+2025-01-03 21:15:40,996 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch diff.patch
+2025-01-03 21:15:40,999 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-03 21:15:40,999 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-03 21:15:40,999 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4
+2025-01-03 21:15:40,999 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4
+2025-01-03 21:15:40,999 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 10
+2025-01-03 21:15:41,000 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit
+2025-01-03 21:15:41,000 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:41,000 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-03 21:15:41,000 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:41,000 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-03 21:15:41,000 INFO    SenderThread:1974795 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-03 21:15:41,154 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json
+2025-01-03 21:15:41,168 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt
+2025-01-03 21:15:41,178 INFO    wandb-upload_0:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml
+2025-01-03 21:15:41,223 INFO    wandb-upload_3:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:41,424 INFO    Thread-11 (_thread_body):1974795 [sender.py:transition_state():614] send defer: 11
+2025-01-03 21:15:41,424 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:41,424 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-03 21:15:41,424 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:41,424 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-03 21:15:41,424 INFO    SenderThread:1974795 [file_pusher.py:join():175] waiting for file pusher
+2025-01-03 21:15:41,425 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 12
+2025-01-03 21:15:41,425 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:41,425 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-03 21:15:41,425 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:41,425 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-03 21:15:41,425 INFO    SenderThread:1974795 [file_stream.py:finish():601] file stream finish called
+2025-01-03 21:15:41,503 INFO    SenderThread:1974795 [file_stream.py:finish():605] file stream finish is done
+2025-01-03 21:15:41,504 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 13
+2025-01-03 21:15:41,504 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:41,504 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-03 21:15:41,504 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:41,504 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-03 21:15:41,504 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 14
+2025-01-03 21:15:41,504 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:41,504 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-03 21:15:41,504 DEBUG   SenderThread:1974795 [sender.py:send():379] send: final
+2025-01-03 21:15:41,505 DEBUG   SenderThread:1974795 [sender.py:send():379] send: footer
+2025-01-03 21:15:41,505 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:41,505 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-03 21:15:41,523 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-03 21:15:41,523 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit
+2025-01-03 21:15:41,525 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:41,527 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-03 21:15:41,527 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-03 21:15:41,529 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-03 21:15:41,529 INFO    HandlerThread:1974795 [handler.py:finish():882] shutting down handler
+2025-01-03 21:15:42,523 INFO    WriterThread:1974795 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb
+2025-01-03 21:15:42,523 INFO    SenderThread:1974795 [sender.py:finish():1615] shutting down sender
+2025-01-03 21:15:42,523 INFO    SenderThread:1974795 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-03 21:15:42,523 INFO    SenderThread:1974795 [file_pusher.py:join():175] waiting for file pusher

48t61yqi/wandb/debug.log

@@ -0,0 +1,37 @@
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Configure stats pid to 1974592
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/settings
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug.log
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug-internal.log
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_init.py:init():569] calling init triggers
+2025-01-03 21:13:07,569 INFO    MainThread:1974592 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi'}
+2025-01-03 21:13:07,569 INFO    MainThread:1974592 [wandb_init.py:init():619] starting backend
+2025-01-03 21:13:07,569 INFO    MainThread:1974592 [wandb_init.py:init():623] setting up manager
+2025-01-03 21:13:07,572 INFO    MainThread:1974592 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-03 21:13:07,574 INFO    MainThread:1974592 [wandb_init.py:init():631] backend started and connected
+2025-01-03 21:13:07,586 INFO    MainThread:1974592 [wandb_init.py:init():720] updated telemetry
+2025-01-03 21:13:07,596 INFO    MainThread:1974592 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-03 21:13:07,960 INFO    MainThread:1974592 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-03 21:13:08,000 INFO    MainThread:1974592 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-03 21:13:08,001 INFO    MainThread:1974592 [wandb_init.py:init():804] starting run threads in backend
+2025-01-03 21:13:08,360 INFO    MainThread:1974592 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-03 21:13:08,360 INFO    MainThread:1974592 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-03 21:13:08,360 INFO    MainThread:1974592 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-03 21:13:08,360 INFO    MainThread:1974592 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-03 21:13:08,361 INFO    MainThread:1974592 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-03 21:13:08,362 INFO    MainThread:1974592 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/tb, True
+2025-01-03 21:13:25,965 INFO    MainThread:1974592 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'}
+2025-01-03 21:15:29,430 INFO    MainThread:1974592 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/48t61yqi
+2025-01-03 21:15:29,433 INFO    MainThread:1974592 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0
+2025-01-03 21:15:29,434 INFO    MainThread:1974592 [wandb_run.py:_restore():2352] restore
+2025-01-03 21:15:29,435 INFO    MainThread:1974592 [wandb_run.py:_restore():2358] restore done
+2025-01-03 21:15:42,533 INFO    MainThread:1974592 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-03 21:15:42,537 INFO    MainThread:1974592 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-03 21:15:42,608 INFO    MainThread:1974592 [wandb_run.py:_footer_sync_info():3975] logging synced files

48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py

@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml

@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 1
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1926_1'
+wandb_notes:
+  desc: null
+  value: 1926_1_restarted_5
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: true
+use_sdf_maps:
+  desc: null
+  value: true
+use_normals_maps:
+  desc: null
+  value: true
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: true
+        use_sdf_maps: true
+        use_normals_maps: true
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 1
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: true
+          use_sdf_maps: true
+          use_normals_maps: true
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1735956787
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 2
+      - 14
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: modelsys
+left_book_slot:
+  desc: null
+  value: twodim

48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch

@@ -0,0 +1,57 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 57a851f..9234106 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -138,12 +138,19 @@ load_checkpoint: ${agent.load_checkpoint}
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1007_0' 
++# wandb_run_id: '1922_1' # seed 1
++# wandb_run_id: '1922_2' # seed 2
++# wandb_run_id: '1938_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+ # wandb_run_id: '1006_0'
++wandb_run_id: '1926_1' # seed 1
++# wandb_run_id: '1930_0' # dataset shuffle seed 1
++
++
+ 
+ # RGBD+mask+contact(contact+DTC)+act history
+-wandb_run_id: '1745_0'
++# wandb_run_id: '1745_0'
+ 
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 71a5d1c..c7eadce 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -159,8 +159,10 @@ run_id_list = [
+ ]
+ 
+ run_id_list = [
+-    # '1006_0',
+-    '1745_0'
++    # '1922_1',
++    # '1922_2',
++    '1926_1',
++    '1930_0',
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py
+index 6f97e26..285afc8 100644
+--- a/gym-envs/gym_envs/envs/hardware/franka.py
++++ b/gym-envs/gym_envs/envs/hardware/franka.py
+@@ -330,8 +330,8 @@ class Franka:
+ 		# self.yaw = yaw
+ 
+ 		self.move_to_target_hz = 10.0
+-		self.move_to_trans_tolerance = 0.005
+-		self.move_to_rot_tolerance = 1.0
++		self.move_to_trans_tolerance = 0.01
++		self.move_to_rot_tolerance = 2.0
+ 		self.move_to_target_max_velocity = 0.8
+ 		self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz)
+ 		self.move_to_target_timeout = 5.0

48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff4d172543858f403e01d883e2e919457e9b67113e1f51ac9eeb1c12fbd6c7b4
+size 1487478

48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log

@@ -0,0 +1,110 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1735956805.969554]: resetting environment
+[INFO] [1735956805.977535]: cleared current plan
+[INFO] [1735956805.978457]: moving to home
+[INFO] [1735956806.980896]: reached home
+[INFO] [1735956806.983056]: reset action history
+[INFO] [1735956809.692550]: environment reset
+Starting episode 0
+[INFO] [1735956809.696665]: resetting environment
+[INFO] [1735956809.704118]: cleared current plan
+[INFO] [1735956809.705085]: moving to home
+[INFO] [1735956810.707855]: reached home
+[INFO] [1735956810.709838]: reset action history
+[INFO] [1735956813.418834]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1735956816.750033]: resetting environment
+[INFO] [1735956816.754664]: cleared current plan
+[INFO] [1735956816.755494]: moving to home
+[INFO] [1735956817.757227]: reached home
+[INFO] [1735956817.759665]: reset action history
+[INFO] [1735956820.466643]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/episode_rosbags/episode_0_2025-01-03-21-13-40.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1974592.log
+started roslaunch server http://158.130.50.37:46869/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [1974904]
+process[print_text-2]: started with pid [1974905]
+process[rosbag_record-3]: started with pid [1974929]
+started bagging!
+For topic gripper_width: timestamp difference is 107834998 for nearest: 1735956846261599961 - target: 1735956846153764963 at idx 249
+For topic gripper_width: timestamp difference is 112545745 for nearest: 1735956852028245465 - target: 1735956851915699720 at idx 249
+[INFO] [1735956855.666153]: Storing episode...
+[rosbag_record-3] killing on exit
+[pub_text-1] killing on exit
+[print_text-2] killing on exit
+[INFO] [1735956856.438961]: Stored episode 1.
+[INFO] [1735956856.439173]: Saving video...
+[INFO] [1735956856.639122]: Video saved!
+Starting episode 1
+[INFO] [1735956856.654068]: resetting environment
+[INFO] [1735956856.663259]: cleared current plan
+[INFO] [1735956856.663538]: moving to home
+preempting episode
+[INFO] [1735956860.366380]: reached home
+[INFO] [1735956860.367252]: reset action history
+[INFO] [1735956863.078712]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1735956896.123101]: resetting environment
+[INFO] [1735956896.134941]: cleared current plan
+[INFO] [1735956896.135912]: moving to home
+[INFO] [1735956897.137384]: reached home
+[INFO] [1735956897.138458]: reset action history
+[INFO] [1735956899.896548]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/episode_rosbags/episode_1_2025-01-03-21-14-59.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1974592.log
+started roslaunch server http://158.130.50.37:39217/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [1975115]
+process[print_text-5]: started with pid [1975116]
+process[rosbag_record-6]: started with pid [1975117]
+started bagging!
+[INFO] [1735956901.286608]: Storing episode...
+[rosbag_record-6] killing on exit[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1735956901.489305]: Stored episode 2.
+[INFO] [1735956901.491349]: Saving video...
+[INFO] [1735956901.528032]: Video saved!
+Starting episode 2
+[INFO] [1735956901.534625]: resetting environment
+[INFO] [1735956901.538407]: cleared current plan
+[INFO] [1735956901.538596]: moving to home
+[INFO] [1735956902.539591]: reached home
+[INFO] [1735956902.542495]: reset action history
+[INFO] [1735956905.319942]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+program exiting gracefully
+proceeding to start episode!
+[INFO] [1735956929.387121]: Saving video...
+[INFO] [1735956929.394713]: Video saved!

48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt

@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-04T02:13:08.025575",
+    "startedAt": "2025-01-04T02:13:07.566812",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "c537eb6734155592809c2a7db5cca971ec186d72"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3750.002625,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.998,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.061,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 804.7593460083008
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}

48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "ff4d172543858f403e01d883e2e919457e9b67113e1f51ac9eeb1c12fbd6c7b4", "size": 1487478, "path": "media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4"}, "global_step": 133, "_timestamp": 1735956896.1220462, "_runtime": 108.54749131202698, "_step": 3, "eval/1_eval": {"_type": "video-file", "sha256": "ae729244ac49b5c344235ffc4fd497edb507b67a3c8a88b5fc613756dc78bb18", "size": 65553, "path": "media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4"}, "eval/video": {"_type": "video-file", "sha256": "25445a41b5bbda0ed7e2e0845eaa5c96373705c5c738fe640c1ea70a07b4176c", "size": 262, "path": "media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4"}, "_wandb": {"runtime": 141}, "eval/num_success": 0.0, "episode": 1.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.9642442464828491, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.9641662836074829, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.944913387298584, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.847197413444519}

48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug-internal.log

@@ -0,0 +1,468 @@
+2025-01-03 21:13:07,574 INFO    StreamThr :1974795 [internal.py:wandb_internal():85] W&B internal server running at pid: 1974795, started at: 2025-01-03 21:13:07.573731
+2025-01-03 21:13:07,576 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status
+2025-01-03 21:13:07,587 INFO    WriterThread:1974795 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb
+2025-01-03 21:13:07,587 DEBUG   SenderThread:1974795 [sender.py:send():379] send: header
+2025-01-03 21:13:07,600 DEBUG   SenderThread:1974795 [sender.py:send():379] send: run
+2025-01-03 21:13:07,945 INFO    SenderThread:1974795 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files
+2025-01-03 21:13:07,945 INFO    SenderThread:1974795 [sender.py:_start_run_threads():1188] run started: 48t61yqi with start time 1735956787.574555
+2025-01-03 21:13:07,961 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: check_version
+2025-01-03 21:13:07,961 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: check_version
+2025-01-03 21:13:08,006 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: run_start
+2025-01-03 21:13:08,011 DEBUG   HandlerThread:1974795 [system_info.py:__init__():26] System info init
+2025-01-03 21:13:08,011 DEBUG   HandlerThread:1974795 [system_info.py:__init__():41] System info init done
+2025-01-03 21:13:08,011 INFO    HandlerThread:1974795 [system_monitor.py:start():194] Starting system monitor
+2025-01-03 21:13:08,011 INFO    SystemMonitor:1974795 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-03 21:13:08,011 INFO    HandlerThread:1974795 [system_monitor.py:probe():214] Collecting system info
+2025-01-03 21:13:08,012 INFO    SystemMonitor:1974795 [interfaces.py:start():188] Started cpu monitoring
+2025-01-03 21:13:08,012 INFO    SystemMonitor:1974795 [interfaces.py:start():188] Started disk monitoring
+2025-01-03 21:13:08,013 INFO    SystemMonitor:1974795 [interfaces.py:start():188] Started gpu monitoring
+2025-01-03 21:13:08,013 INFO    SystemMonitor:1974795 [interfaces.py:start():188] Started memory monitoring
+2025-01-03 21:13:08,013 INFO    SystemMonitor:1974795 [interfaces.py:start():188] Started network monitoring
+2025-01-03 21:13:08,025 DEBUG   HandlerThread:1974795 [system_info.py:probe():152] Probing system
+2025-01-03 21:13:08,027 DEBUG   HandlerThread:1974795 [system_info.py:_probe_git():137] Probing git
+2025-01-03 21:13:08,031 DEBUG   HandlerThread:1974795 [system_info.py:_probe_git():145] Probing git done
+2025-01-03 21:13:08,031 DEBUG   HandlerThread:1974795 [system_info.py:probe():200] Probing system done
+2025-01-03 21:13:08,031 DEBUG   HandlerThread:1974795 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T02:13:08.025575', 'startedAt': '2025-01-04T02:13:07.566812', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3750.002625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.061, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 804.7593460083008}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-03 21:13:08,031 INFO    HandlerThread:1974795 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-03 21:13:08,031 INFO    HandlerThread:1974795 [system_monitor.py:probe():227] Publishing system info
+2025-01-03 21:13:08,031 DEBUG   HandlerThread:1974795 [system_info.py:_save_code():44] Saving code
+2025-01-03 21:13:08,033 DEBUG   HandlerThread:1974795 [system_info.py:_save_code():67] Saving code done
+2025-01-03 21:13:08,033 DEBUG   HandlerThread:1974795 [system_info.py:_save_patches():84] Saving git patches
+2025-01-03 21:13:08,094 DEBUG   HandlerThread:1974795 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-03 21:13:08,095 INFO    HandlerThread:1974795 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-03 21:13:08,098 DEBUG   SenderThread:1974795 [sender.py:send():379] send: files
+2025-01-03 21:13:08,098 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-03 21:13:08,098 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-03 21:13:08,098 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-03 21:13:08,340 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/7zaf7m84-diff.patch
+2025-01-03 21:13:08,350 INFO    wandb-upload_0:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/2v3neepq-wandb-metadata.json
+2025-01-03 21:13:08,359 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-03 21:13:08,360 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: python_packages
+2025-01-03 21:13:08,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:13:08,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:08,361 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:13:08,362 INFO    HandlerThread:1974795 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/tb"
+  save: true
+}
+control {
+  relay_id: "106e59ccfedd40bd8337b716b182d46a"
+}
+_info {
+  stream_id: "48t61yqi"
+}
+
+2025-01-03 21:13:08,362 DEBUG   HandlerThread:1974795 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-03 21:13:08,373 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/wfd4dtnr-code/FISH/eval_robot.py
+2025-01-03 21:13:08,515 DEBUG   SenderThread:1974795 [sender.py:send():379] send: telemetry
+2025-01-03 21:13:08,515 DEBUG   SenderThread:1974795 [sender.py:send():379] send: tbrecord
+2025-01-03 21:13:08,515 DEBUG   SenderThread:1974795 [sender.py:send():379] send: files
+2025-01-03 21:13:08,515 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 with policy live
+2025-01-03 21:13:08,698 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/utluf6qk-events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0
+2025-01-03 21:13:08,946 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt
+2025-01-03 21:13:08,946 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH
+2025-01-03 21:13:08,947 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code
+2025-01-03 21:13:09,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:10,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:10,945 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:11,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:12,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:13,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:13,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:14,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:15,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:16,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:17,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:18,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:18,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:19,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:20,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:21,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:22,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:23,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:13:23,360 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:13:23,401 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:23,464 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:24,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:25,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:25,966 DEBUG   SenderThread:1974795 [sender.py:send():379] send: config
+2025-01-03 21:13:26,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:26,950 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:27,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:28,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:28,950 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:29,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:29,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:30,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:30,951 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:31,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:32,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:32,951 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:33,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:34,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:34,461 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:34,952 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:35,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:36,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:37,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:38,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:13:38,360 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:13:38,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:38,953 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:39,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:39,520 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:39,953 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml
+2025-01-03 21:13:40,364 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:41,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:42,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:42,954 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:13:43,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:44,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:45,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:45,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:46,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:47,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:48,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:49,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:50,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:50,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:51,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:52,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:53,360 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:13:53,360 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:13:53,401 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:54,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:55,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:55,497 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:13:56,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:57,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:58,366 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:13:59,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:00,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:01,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:01,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:02,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:03,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:04,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:05,367 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:06,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:07,313 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:07,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:08,014 DEBUG   SystemMonitor:1974795 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-03 21:14:08,014 DEBUG   SenderThread:1974795 [sender.py:send():379] send: stats
+2025-01-03 21:14:08,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:14:08,361 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:14:08,409 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:08,961 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:09,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:10,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:11,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:12,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:12,962 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:13,073 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:13,368 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:14,372 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:15,370 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:16,371 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:16,652 DEBUG   SenderThread:1974795 [sender.py:send():379] send: files
+2025-01-03 21:14:16,652 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 with policy now
+2025-01-03 21:14:16,652 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-03 21:14:16,653 DEBUG   SenderThread:1974795 [sender.py:send():379] send: history
+2025-01-03 21:14:16,654 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record
+2025-01-03 21:14:16,654 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:14:16,878 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/kh7pplhk-media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4
+2025-01-03 21:14:16,963 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:16,963 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4
+2025-01-03 21:14:16,964 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json
+2025-01-03 21:14:16,964 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos
+2025-01-03 21:14:16,964 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media
+2025-01-03 21:14:16,964 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval
+2025-01-03 21:14:17,232 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/rsn814pa-media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4
+2025-01-03 21:14:17,371 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:18,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:18,372 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:18,964 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:19,373 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:20,409 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:21,372 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:22,374 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:22,965 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:23,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:14:23,361 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:14:23,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:23,479 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:24,374 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:24,966 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:25,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:26,374 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:27,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:28,374 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:29,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:29,374 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:30,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:31,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:32,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:33,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:34,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:35,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:35,375 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:36,376 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:37,376 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:38,015 DEBUG   SenderThread:1974795 [sender.py:send():379] send: stats
+2025-01-03 21:14:38,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:14:38,361 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:14:38,405 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:39,376 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:40,376 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:40,473 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:41,377 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:42,377 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:43,377 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:44,378 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:45,377 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:46,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:46,377 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:47,378 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:48,378 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:49,378 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:50,379 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:51,379 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:52,363 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:52,379 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:53,361 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:14:53,362 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:14:53,409 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:54,379 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:55,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:56,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:56,976 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:56,976 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0
+2025-01-03 21:14:57,223 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/2qynfcuc-events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0
+2025-01-03 21:14:57,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:58,181 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:14:58,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:14:58,976 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:14:59,381 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:00,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:00,977 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:01,380 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:01,533 DEBUG   SenderThread:1974795 [sender.py:send():379] send: files
+2025-01-03 21:15:01,533 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 with policy now
+2025-01-03 21:15:01,533 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-03 21:15:01,534 DEBUG   SenderThread:1974795 [sender.py:send():379] send: history
+2025-01-03 21:15:01,534 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record
+2025-01-03 21:15:01,535 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:15:01,978 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json
+2025-01-03 21:15:01,978 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4
+2025-01-03 21:15:01,978 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval
+2025-01-03 21:15:02,264 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/ob86mf5s-media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4
+2025-01-03 21:15:02,389 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:02,513 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/c7fg0d8r-media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4
+2025-01-03 21:15:02,978 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:03,365 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:03,389 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:04,384 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:04,979 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:05,383 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:06,385 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:06,979 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:07,384 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:08,015 DEBUG   SenderThread:1974795 [sender.py:send():379] send: stats
+2025-01-03 21:15:08,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:15:08,363 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:15:08,405 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:08,465 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:09,388 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:10,389 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:11,388 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:12,390 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:13,393 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:14,369 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:14,390 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:15,393 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:16,391 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:17,396 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:18,396 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:19,372 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:19,395 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:20,395 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:21,395 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:22,395 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:23,362 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-03 21:15:23,362 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: stop_status
+2025-01-03 21:15:23,409 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:24,397 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:24,467 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:25,399 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:26,399 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:27,399 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:28,401 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:29,403 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:29,420 DEBUG   SenderThread:1974795 [sender.py:send():379] send: files
+2025-01-03 21:15:29,420 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 with policy now
+2025-01-03 21:15:29,465 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-03 21:15:29,465 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: server_info
+2025-01-03 21:15:29,466 DEBUG   SenderThread:1974795 [sender.py:send():379] send: history
+2025-01-03 21:15:29,466 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record
+2025-01-03 21:15:29,467 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:15:29,468 DEBUG   SenderThread:1974795 [sender.py:send():379] send: telemetry
+2025-01-03 21:15:29,468 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: server_info
+2025-01-03 21:15:29,468 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:29,537 DEBUG   SenderThread:1974795 [sender.py:send():379] send: exit
+2025-01-03 21:15:29,537 INFO    SenderThread:1974795 [sender.py:send_exit():586] handling exit code: 0
+2025-01-03 21:15:29,537 INFO    SenderThread:1974795 [sender.py:send_exit():588] handling runtime: 141
+2025-01-03 21:15:29,537 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:15:29,537 INFO    SenderThread:1974795 [sender.py:send_exit():594] send defer
+2025-01-03 21:15:29,537 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:29,537 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-03 21:15:29,538 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:29,538 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-03 21:15:29,538 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 1
+2025-01-03 21:15:29,538 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:29,538 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-03 21:15:29,538 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:29,538 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-03 21:15:29,538 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 2
+2025-01-03 21:15:29,538 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:29,538 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-03 21:15:29,538 INFO    HandlerThread:1974795 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-03 21:15:29,538 DEBUG   SystemMonitor:1974795 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-03 21:15:29,538 INFO    HandlerThread:1974795 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-03 21:15:29,538 DEBUG   SystemMonitor:1974795 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-03 21:15:29,539 INFO    HandlerThread:1974795 [interfaces.py:finish():200] Joined disk monitor
+2025-01-03 21:15:29,590 INFO    HandlerThread:1974795 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-03 21:15:29,591 INFO    HandlerThread:1974795 [interfaces.py:finish():200] Joined memory monitor
+2025-01-03 21:15:29,591 INFO    HandlerThread:1974795 [interfaces.py:finish():200] Joined network monitor
+2025-01-03 21:15:29,591 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:29,591 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-03 21:15:29,591 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 3
+2025-01-03 21:15:29,591 DEBUG   SenderThread:1974795 [sender.py:send():379] send: stats
+2025-01-03 21:15:29,591 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:29,591 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-03 21:15:29,592 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:29,592 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-03 21:15:29,592 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 4
+2025-01-03 21:15:29,592 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:29,592 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-03 21:15:29,596 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/44ln0e3q-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-03 21:15:29,987 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json
+2025-01-03 21:15:29,987 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-03 21:15:29,987 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval
+2025-01-03 21:15:30,242 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /tmp/tmpm2wbqrblwandb/vh0f04q3-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-03 21:15:30,987 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:36,759 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-03 21:15:36,760 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:36,760 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-03 21:15:36,760 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:36,761 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-03 21:15:36,761 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 5
+2025-01-03 21:15:36,761 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit
+2025-01-03 21:15:36,761 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:36,761 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-03 21:15:36,762 DEBUG   SenderThread:1974795 [sender.py:send():379] send: history
+2025-01-03 21:15:36,762 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: summary_record
+2025-01-03 21:15:36,762 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:15:36,762 DEBUG   SenderThread:1974795 [sender.py:send():379] send: summary
+2025-01-03 21:15:36,766 INFO    SenderThread:1974795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-03 21:15:36,767 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:36,767 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-03 21:15:36,767 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 6
+2025-01-03 21:15:36,767 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:36,767 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-03 21:15:36,767 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:36,767 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-03 21:15:36,770 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: status_report
+2025-01-03 21:15:36,885 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 7
+2025-01-03 21:15:36,885 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:36,885 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-03 21:15:36,885 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:36,885 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-03 21:15:36,989 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml
+2025-01-03 21:15:36,989 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json
+2025-01-03 21:15:37,544 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-03 21:15:38,663 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 8
+2025-01-03 21:15:38,664 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit
+2025-01-03 21:15:38,664 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:38,664 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-03 21:15:38,664 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:38,664 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-03 21:15:38,664 INFO    SenderThread:1974795 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-03 21:15:38,666 INFO    SenderThread:1974795 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-03 21:15:38,680 INFO    SenderThread:1974795 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-03 21:15:38,694 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 9
+2025-01-03 21:15:38,694 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:38,694 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-03 21:15:38,694 DEBUG   SenderThread:1974795 [sender.py:send():379] send: artifact
+2025-01-03 21:15:38,990 INFO    Thread-12 :1974795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:39,322 INFO    wandb-upload_0:1974795 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjy2pywco
+2025-01-03 21:15:39,428 INFO    wandb-upload_2:1974795 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqkmuwojg
+2025-01-03 21:15:39,482 INFO    wandb-upload_1:1974795 [upload_job.py:push():88] Uploaded file /tmp/tmpwr759ffl/wandb-job.json
+2025-01-03 21:15:39,558 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-03 21:15:40,071 INFO    SenderThread:1974795 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg5OTM1NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'versionIndex': 408}}}
+2025-01-03 21:15:40,072 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:40,072 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-03 21:15:40,072 INFO    SenderThread:1974795 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-03 21:15:40,991 INFO    SenderThread:1974795 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files
+2025-01-03 21:15:40,991 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml config.yaml
+2025-01-03 21:15:40,991 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json wandb-summary.json
+2025-01-03 21:15:40,991 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt requirements.txt
+2025-01-03 21:15:40,991 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-metadata.json wandb-metadata.json
+2025-01-03 21:15:40,993 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0 events.out.tfevents.1735956788.leonmkim-ROG-Strix-G15CS-G15CS.1974592.0
+2025-01-03 21:15:40,996 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log output.log
+2025-01-03 21:15:40,996 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/diff.patch diff.patch
+2025-01-03 21:15:40,999 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-03 21:15:40,999 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-03 21:15:40,999 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4 media/videos/eval/0_eval_0_ff4d172543858f403e01.mp4
+2025-01-03 21:15:40,999 INFO    SenderThread:1974795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4 media/videos/eval/1_eval_1_ae729244ac49b5c34423.mp4
+2025-01-03 21:15:40,999 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 10
+2025-01-03 21:15:41,000 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit
+2025-01-03 21:15:41,000 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:41,000 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-03 21:15:41,000 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:41,000 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-03 21:15:41,000 INFO    SenderThread:1974795 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-03 21:15:41,154 INFO    wandb-upload_2:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/wandb-summary.json
+2025-01-03 21:15:41,168 INFO    wandb-upload_1:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/requirements.txt
+2025-01-03 21:15:41,178 INFO    wandb-upload_0:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/config.yaml
+2025-01-03 21:15:41,223 INFO    wandb-upload_3:1974795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/files/output.log
+2025-01-03 21:15:41,424 INFO    Thread-11 (_thread_body):1974795 [sender.py:transition_state():614] send defer: 11
+2025-01-03 21:15:41,424 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:41,424 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-03 21:15:41,424 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:41,424 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-03 21:15:41,424 INFO    SenderThread:1974795 [file_pusher.py:join():175] waiting for file pusher
+2025-01-03 21:15:41,425 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 12
+2025-01-03 21:15:41,425 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:41,425 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-03 21:15:41,425 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:41,425 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-03 21:15:41,425 INFO    SenderThread:1974795 [file_stream.py:finish():601] file stream finish called
+2025-01-03 21:15:41,503 INFO    SenderThread:1974795 [file_stream.py:finish():605] file stream finish is done
+2025-01-03 21:15:41,504 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 13
+2025-01-03 21:15:41,504 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:41,504 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-03 21:15:41,504 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:41,504 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-03 21:15:41,504 INFO    SenderThread:1974795 [sender.py:transition_state():614] send defer: 14
+2025-01-03 21:15:41,504 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: defer
+2025-01-03 21:15:41,504 INFO    HandlerThread:1974795 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-03 21:15:41,504 DEBUG   SenderThread:1974795 [sender.py:send():379] send: final
+2025-01-03 21:15:41,505 DEBUG   SenderThread:1974795 [sender.py:send():379] send: footer
+2025-01-03 21:15:41,505 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: defer
+2025-01-03 21:15:41,505 INFO    SenderThread:1974795 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-03 21:15:41,523 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-03 21:15:41,523 DEBUG   SenderThread:1974795 [sender.py:send_request():406] send_request: poll_exit
+2025-01-03 21:15:41,525 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-03 21:15:41,527 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-03 21:15:41,527 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-03 21:15:41,529 DEBUG   HandlerThread:1974795 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-03 21:15:41,529 INFO    HandlerThread:1974795 [handler.py:finish():882] shutting down handler
+2025-01-03 21:15:42,523 INFO    WriterThread:1974795 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/run-48t61yqi.wandb
+2025-01-03 21:15:42,523 INFO    SenderThread:1974795 [sender.py:finish():1615] shutting down sender
+2025-01-03 21:15:42,523 INFO    SenderThread:1974795 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-03 21:15:42,523 INFO    SenderThread:1974795 [file_pusher.py:join():175] waiting for file pusher

48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug.log

@@ -0,0 +1,37 @@
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Configure stats pid to 1974592
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/settings
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug.log
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/wandb/run-20250103_211307-48t61yqi/logs/debug-internal.log
+2025-01-03 21:13:07,568 INFO    MainThread:1974592 [wandb_init.py:init():569] calling init triggers
+2025-01-03 21:13:07,569 INFO    MainThread:1974592 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi'}
+2025-01-03 21:13:07,569 INFO    MainThread:1974592 [wandb_init.py:init():619] starting backend
+2025-01-03 21:13:07,569 INFO    MainThread:1974592 [wandb_init.py:init():623] setting up manager
+2025-01-03 21:13:07,572 INFO    MainThread:1974592 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-03 21:13:07,574 INFO    MainThread:1974592 [wandb_init.py:init():631] backend started and connected
+2025-01-03 21:13:07,586 INFO    MainThread:1974592 [wandb_init.py:init():720] updated telemetry
+2025-01-03 21:13:07,596 INFO    MainThread:1974592 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-03 21:13:07,960 INFO    MainThread:1974592 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-03 21:13:08,000 INFO    MainThread:1974592 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-03 21:13:08,001 INFO    MainThread:1974592 [wandb_init.py:init():804] starting run threads in backend
+2025-01-03 21:13:08,360 INFO    MainThread:1974592 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-03 21:13:08,360 INFO    MainThread:1974592 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-03 21:13:08,360 INFO    MainThread:1974592 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-03 21:13:08,360 INFO    MainThread:1974592 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-03 21:13:08,361 INFO    MainThread:1974592 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-03 21:13:08,362 INFO    MainThread:1974592 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/48t61yqi/tb, True
+2025-01-03 21:13:25,965 INFO    MainThread:1974592 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'}
+2025-01-03 21:15:29,430 INFO    MainThread:1974592 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/48t61yqi
+2025-01-03 21:15:29,433 INFO    MainThread:1974592 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0
+2025-01-03 21:15:29,434 INFO    MainThread:1974592 [wandb_run.py:_restore():2352] restore
+2025-01-03 21:15:29,435 INFO    MainThread:1974592 [wandb_run.py:_restore():2358] restore done
+2025-01-03 21:15:42,533 INFO    MainThread:1974592 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-03 21:15:42,537 INFO    MainThread:1974592 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-03 21:15:42,608 INFO    MainThread:1974592 [wandb_run.py:_footer_sync_info():3975] logging synced files

dqnz7k8g/eval_policy.log

@@ -0,0 +1,15 @@
+[2025-01-03 22:52:20,393][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-03 22:52:20,397][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-03 22:52:23,976][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

iqjkgl2e/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1926_1'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

iqjkgl2e/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

iqjkgl2e/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

iqjkgl2e/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

iqjkgl2e/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

iqjkgl2e/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

iqjkgl2e/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

iqjkgl2e/eval_robot.log

@@ -0,0 +1,12 @@
+[2025-01-04 22:28:33,145][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-04 22:28:33,148][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

iqjkgl2e/eval_video/0_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f829840c9d5231f7c2a1ef7d0f2ce345f32e53845f431d6604f993f0feabd02
+size 1518861

iqjkgl2e/eval_video/1_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcc2c84034b2f80493e2c9bcfdfe89f5533ab66e5acaa9e5a19c94d4ab517e9c
+size 1504602

iqjkgl2e/eval_video/2_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fef06f9228f28f1375655e2e1fa944ee49dc23e690f3c86ce1bc64db99b990e5
+size 1528137

iqjkgl2e/eval_video/3_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ceb841f3e55434b9bc762ea3190b2ad5c6886d8ad25c7c25631e6f71967352c
+size 1544200

iqjkgl2e/eval_video/4_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d691d8c6f8564711e47c11a3353dd90ba7d94653918a5c63f47690e8a8b9c0ae
+size 1542124

iqjkgl2e/tb/events.out.tfevents.1736047719.leonmkim-ROG-Strix-G15CS-G15CS.2037421.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21bf7564b93ce583dfb9f81a429595fbae0b4760f63c86d77529d4cb089ca31b
+size 6539

iqjkgl2e/wandb/debug.log

@@ -0,0 +1,31 @@
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_setup.py:_flush():76] Configure stats pid to 2037421
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/settings
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug.log
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/logs/debug-internal.log
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_init.py:init():569] calling init triggers
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1926_1', 'wandb_notes': '1926_1_restarted_5', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e'}
+2025-01-04 22:28:38,817 INFO    MainThread:2037421 [wandb_init.py:init():619] starting backend
+2025-01-04 22:28:38,818 INFO    MainThread:2037421 [wandb_init.py:init():623] setting up manager
+2025-01-04 22:28:38,822 INFO    MainThread:2037421 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-04 22:28:38,822 INFO    MainThread:2037421 [wandb_init.py:init():631] backend started and connected
+2025-01-04 22:28:38,835 INFO    MainThread:2037421 [wandb_init.py:init():720] updated telemetry
+2025-01-04 22:28:38,843 INFO    MainThread:2037421 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-04 22:28:39,090 INFO    MainThread:2037421 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-04 22:28:39,131 INFO    MainThread:2037421 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-04 22:28:39,132 INFO    MainThread:2037421 [wandb_init.py:init():804] starting run threads in backend
+2025-01-04 22:28:39,488 INFO    MainThread:2037421 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-04 22:28:39,488 INFO    MainThread:2037421 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-04 22:28:39,488 INFO    MainThread:2037421 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-04 22:28:39,489 INFO    MainThread:2037421 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-04 22:28:39,490 INFO    MainThread:2037421 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-04 22:28:39,490 INFO    MainThread:2037421 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e/tb, True
+2025-01-04 22:28:52,929 INFO    MainThread:2037421 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2025-01-04 22:34:39,795 WARNING MsgRouterThr:2037421 [router.py:message_loop():77] message_loop has been closed

iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/code/FISH/eval_robot.py

@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/config.yaml

@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 1
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1926_1'
+wandb_notes:
+  desc: null
+  value: 1926_1_restarted_5
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: true
+use_sdf_maps:
+  desc: null
+  value: true
+use_normals_maps:
+  desc: null
+  value: true
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 7
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: true
+        use_sdf_maps: true
+        use_normals_maps: true
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 1
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: true
+          use_sdf_maps: true
+          use_normals_maps: true
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1926_1/iqjkgl2e
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1736047718
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 14
+      - 16
+      - 23
+      - 35
+      - 62
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: hbm
+left_book_slot:
+  desc: null
+  value: twodim

iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/diff.patch

@@ -0,0 +1,56 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index 57a851f..27e7d93 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint}
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1007_0' 
++# wandb_run_id: '1922_1' # seed 1
++# wandb_run_id: '1938_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+ # wandb_run_id: '1006_0'
++wandb_run_id: '1926_1' # seed 1
++# wandb_run_id: '1930_0' # dataset shuffle seed 1
++
++
+ 
+ # RGBD+mask+contact(contact+DTC)+act history
+-wandb_run_id: '1745_0'
++# wandb_run_id: '1745_0'
+ 
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 71a5d1c..c7eadce 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -159,8 +159,10 @@ run_id_list = [
+ ]
+ 
+ run_id_list = [
+-    # '1006_0',
+-    '1745_0'
++    # '1922_1',
++    # '1922_2',
++    '1926_1',
++    '1930_0',
+ ]
+ 
+ checkpoint_epoch = 12000
+diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py
+index 6f97e26..285afc8 100644
+--- a/gym-envs/gym_envs/envs/hardware/franka.py
++++ b/gym-envs/gym_envs/envs/hardware/franka.py
+@@ -330,8 +330,8 @@ class Franka:
+ 		# self.yaw = yaw
+ 
+ 		self.move_to_target_hz = 10.0
+-		self.move_to_trans_tolerance = 0.005
+-		self.move_to_rot_tolerance = 1.0
++		self.move_to_trans_tolerance = 0.01
++		self.move_to_rot_tolerance = 2.0
+ 		self.move_to_target_max_velocity = 0.8
+ 		self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz)
+ 		self.move_to_target_timeout = 5.0

iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/success_rate_ci_5_2d447e7da5390d365153.table.json

@@ -0,0 +1 @@
+{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.023880511617516982, 0.7333634495268442, 0.015644066132618713, 0.6888955831149919, 0.02994169304235303, 0.5159374773470475, 0.021239768809464744, 0.4524847120141646]]}

iqjkgl2e/wandb/run-20250104_222838-iqjkgl2e/files/media/table/eval/total_success_rate_ci_6_157933031db0c923cb36.table.json

@@ -0,0 +1 @@
+{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[27, 60, 0.45, 0.3149867058170319, 0.5956960320281325, 0.3247884661309667, 0.5717679709052554, 0.34116975221234236, 0.5574955567602561, 0.37607003378366466, 0.5365587621854144]]}