Add files using upload-large-folder tool

.gitattributes

@@ -104,3 +104,43 @@ yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/0_eval_0_900
 yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 filter=lfs diff=lfs merge=lfs -text
 yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 filter=lfs diff=lfs merge=lfs -text
 yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 filter=lfs diff=lfs merge=lfs -text
+1evqtpqc/episode_rosbags/episode_12_2025-01-07-23-20-06.bag filter=lfs diff=lfs merge=lfs -text
+yyf8aq3v/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+yyf8aq3v/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+yyf8aq3v/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+yyf8aq3v/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+yyf8aq3v/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+you6ychq/episode_rosbags/episode_1_2025-01-07-17-42-15.bag filter=lfs diff=lfs merge=lfs -text
+1evqtpqc/episode_rosbags/episode_4_2025-01-07-23-12-53.bag filter=lfs diff=lfs merge=lfs -text
+1evqtpqc/episode_rosbags/episode_17_2025-01-07-23-24-22.bag filter=lfs diff=lfs merge=lfs -text
+jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/2_eval_2_4bd59ccb6d36474a6990.mp4 filter=lfs diff=lfs merge=lfs -text
+jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/3_eval_3_4804a8e4dbe4f9be433d.mp4 filter=lfs diff=lfs merge=lfs -text
+jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/0_eval_0_88c751aa70732cb0acb7.mp4 filter=lfs diff=lfs merge=lfs -text
+jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/4_eval_4_c3747e786ba19b8c5b5a.mp4 filter=lfs diff=lfs merge=lfs -text
+jqiikotd/wandb/run-20250107_172307-jqiikotd/files/media/videos/eval/1_eval_1_3ac128dda904a091cb30.mp4 filter=lfs diff=lfs merge=lfs -text
+jqiikotd/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+jqiikotd/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+jqiikotd/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+jqiikotd/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+jqiikotd/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+1evqtpqc/episode_rosbags/episode_11_2025-01-07-23-19-17.bag filter=lfs diff=lfs merge=lfs -text
+yyf8aq3v/episode_rosbags/episode_1_2025-01-07-17-37-48.bag filter=lfs diff=lfs merge=lfs -text
+1evqtpqc/episode_rosbags/episode_0_2025-01-07-23-09-46.bag filter=lfs diff=lfs merge=lfs -text
+j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/2_eval_2_5f397337911c3cdb0417.mp4 filter=lfs diff=lfs merge=lfs -text
+j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/0_eval_0_51cd14c47da033401fce.mp4 filter=lfs diff=lfs merge=lfs -text
+j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/3_eval_3_67d1471127c440b7c93b.mp4 filter=lfs diff=lfs merge=lfs -text
+j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/4_eval_4_fdfd37d3b1b9058b2a84.mp4 filter=lfs diff=lfs merge=lfs -text
+yyf8aq3v/episode_rosbags/episode_2_2025-01-07-17-38-32.bag filter=lfs diff=lfs merge=lfs -text
+j5vura5m/wandb/run-20250107_171833-j5vura5m/files/media/videos/eval/1_eval_1_7111b6d98d7ffa6a81ae.mp4 filter=lfs diff=lfs merge=lfs -text
+yyf8aq3v/episode_rosbags/episode_0_2025-01-07-17-37-08.bag filter=lfs diff=lfs merge=lfs -text
+j5vura5m/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+j5vura5m/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+j5vura5m/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+j5vura5m/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+j5vura5m/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 filter=lfs diff=lfs merge=lfs -text
+3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 filter=lfs diff=lfs merge=lfs -text
+3b422grv/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+3b422grv/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+1evqtpqc/episode_rosbags/episode_3_2025-01-07-23-12-05.bag filter=lfs diff=lfs merge=lfs -text
+j5vura5m/episode_rosbags/episode_2_2025-01-07-17-20-22.bag filter=lfs diff=lfs merge=lfs -text

1evqtpqc/episode_rosbags/episode_0_2025-01-07-23-09-46.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f78feed4ff8a94dfba4a23e5df9eeca71ee1aa7dcbcd4eb78d08e2c8cf128e2
+size 1412860245

1evqtpqc/episode_rosbags/episode_11_2025-01-07-23-19-17.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20c43652a124c993d3ed9900e484f3e426829dacb450108c1628add94f6b39b1
+size 1410434229

1evqtpqc/episode_rosbags/episode_12_2025-01-07-23-20-06.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35dd3627c1fca76f3d56481b1d25717179be2c20c14f9664959103ac0d4cc9c2
+size 1406647698

1evqtpqc/episode_rosbags/episode_17_2025-01-07-23-24-22.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bdacdadedf09f281cc585c44af876197c24da8d49d569e6e7889c57d2eb30e7
+size 1412521715

1evqtpqc/episode_rosbags/episode_3_2025-01-07-23-12-05.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d04eaee91038b1ebd4b809e1f733af0f8bb40b700421f936e4673967b45b4a64
+size 1412876833

1evqtpqc/episode_rosbags/episode_4_2025-01-07-23-12-53.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7854c4749bf757dbc4a8ada76206013ed99fd944a1830538b9685a4813c5f81
+size 1411637012

3b422grv/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1002_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

3b422grv/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

3b422grv/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

3b422grv/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

3b422grv/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

3b422grv/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

3b422grv/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

3b422grv/eval_robot.log

@@ -0,0 +1,12 @@
+[2025-01-06 17:03:07,239][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-06 17:03:07,242][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

3b422grv/eval_video/0_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d381be8fc37a39636946143ce961a0e9726325f87b53c10b796bff7cecf2908b
+size 1203871

3b422grv/eval_video/1_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43ca52fa14eb631b4abc059d2e4d2836c530e82cde9bbebfb4166f6247188f98
+size 1121306

3b422grv/tb/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af97acafbfeefa218b28bf11b31f7969fa610d6f6080ad82ca33ee336e89901d
+size 1171

3b422grv/wandb/debug-internal.log

@@ -0,0 +1,545 @@
+2025-01-06 17:03:13,347 INFO    StreamThr :2084424 [internal.py:wandb_internal():85] W&B internal server running at pid: 2084424, started at: 2025-01-06 17:03:13.347290
+2025-01-06 17:03:13,348 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status
+2025-01-06 17:03:13,360 INFO    WriterThread:2084424 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb
+2025-01-06 17:03:13,360 DEBUG   SenderThread:2084424 [sender.py:send():379] send: header
+2025-01-06 17:03:13,367 DEBUG   SenderThread:2084424 [sender.py:send():379] send: run
+2025-01-06 17:03:13,578 INFO    SenderThread:2084424 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files
+2025-01-06 17:03:13,578 INFO    SenderThread:2084424 [sender.py:_start_run_threads():1188] run started: 3b422grv with start time 1736200993.348662
+2025-01-06 17:03:13,599 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: check_version
+2025-01-06 17:03:13,599 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: check_version
+2025-01-06 17:03:13,703 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: run_start
+2025-01-06 17:03:13,713 DEBUG   HandlerThread:2084424 [system_info.py:__init__():26] System info init
+2025-01-06 17:03:13,714 DEBUG   HandlerThread:2084424 [system_info.py:__init__():41] System info init done
+2025-01-06 17:03:13,714 INFO    HandlerThread:2084424 [system_monitor.py:start():194] Starting system monitor
+2025-01-06 17:03:13,714 INFO    SystemMonitor:2084424 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-06 17:03:13,714 INFO    HandlerThread:2084424 [system_monitor.py:probe():214] Collecting system info
+2025-01-06 17:03:13,715 INFO    SystemMonitor:2084424 [interfaces.py:start():188] Started cpu monitoring
+2025-01-06 17:03:13,716 INFO    SystemMonitor:2084424 [interfaces.py:start():188] Started disk monitoring
+2025-01-06 17:03:13,716 INFO    SystemMonitor:2084424 [interfaces.py:start():188] Started gpu monitoring
+2025-01-06 17:03:13,717 INFO    SystemMonitor:2084424 [interfaces.py:start():188] Started memory monitoring
+2025-01-06 17:03:13,717 INFO    SystemMonitor:2084424 [interfaces.py:start():188] Started network monitoring
+2025-01-06 17:03:13,727 DEBUG   HandlerThread:2084424 [system_info.py:probe():152] Probing system
+2025-01-06 17:03:13,728 DEBUG   HandlerThread:2084424 [system_info.py:_probe_git():137] Probing git
+2025-01-06 17:03:13,733 DEBUG   HandlerThread:2084424 [system_info.py:_probe_git():145] Probing git done
+2025-01-06 17:03:13,733 DEBUG   HandlerThread:2084424 [system_info.py:probe():200] Probing system done
+2025-01-06 17:03:13,733 DEBUG   HandlerThread:2084424 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:03:13.727505', 'startedAt': '2025-01-06T22:03:13.338476', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3375.356375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.947, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 311.6428985595703}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-06 17:03:13,733 INFO    HandlerThread:2084424 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-06 17:03:13,733 INFO    HandlerThread:2084424 [system_monitor.py:probe():227] Publishing system info
+2025-01-06 17:03:13,733 DEBUG   HandlerThread:2084424 [system_info.py:_save_code():44] Saving code
+2025-01-06 17:03:13,735 DEBUG   HandlerThread:2084424 [system_info.py:_save_code():67] Saving code done
+2025-01-06 17:03:13,735 DEBUG   HandlerThread:2084424 [system_info.py:_save_patches():84] Saving git patches
+2025-01-06 17:03:13,796 DEBUG   HandlerThread:2084424 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-06 17:03:13,797 INFO    HandlerThread:2084424 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-06 17:03:13,800 DEBUG   SenderThread:2084424 [sender.py:send():379] send: files
+2025-01-06 17:03:13,800 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-06 17:03:13,800 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-06 17:03:13,800 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-06 17:03:14,054 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/v4bqqb1x-wandb-metadata.json
+2025-01-06 17:03:14,062 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-06 17:03:14,063 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: python_packages
+2025-01-06 17:03:14,064 DEBUG   SenderThread:2084424 [sender.py:send():379] send: telemetry
+2025-01-06 17:03:14,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:03:14,065 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:03:14,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:14,066 INFO    HandlerThread:2084424 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/tb"
+  save: true
+}
+control {
+  relay_id: "b0cd36502b184c79a585690d8b1a1e21"
+}
+_info {
+  stream_id: "3b422grv"
+}
+
+2025-01-06 17:03:14,066 DEBUG   HandlerThread:2084424 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-06 17:03:14,071 INFO    wandb-upload_2:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/axgmigee-diff.patch
+2025-01-06 17:03:14,075 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/rp2vzrhc-code/FISH/eval_robot.py
+2025-01-06 17:03:14,219 DEBUG   SenderThread:2084424 [sender.py:send():379] send: tbrecord
+2025-01-06 17:03:14,219 DEBUG   SenderThread:2084424 [sender.py:send():379] send: files
+2025-01-06 17:03:14,219 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 with policy live
+2025-01-06 17:03:14,395 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/dk5st5le-events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0
+2025-01-06 17:03:14,580 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py
+2025-01-06 17:03:14,580 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:14,580 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0
+2025-01-06 17:03:14,580 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt
+2025-01-06 17:03:14,581 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch
+2025-01-06 17:03:14,581 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json
+2025-01-06 17:03:14,581 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH
+2025-01-06 17:03:14,581 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code
+2025-01-06 17:03:15,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:16,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:16,579 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:17,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:18,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:19,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:19,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:20,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:20,151 DEBUG   SenderThread:2084424 [sender.py:send():379] send: config
+2025-01-06 17:03:21,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:22,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:22,581 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:23,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:24,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:24,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:25,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:26,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:26,582 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:27,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:28,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:29,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:03:29,063 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:03:29,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:29,205 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:30,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:30,583 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:31,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:32,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:32,584 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:33,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:34,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:34,394 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:35,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:36,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:36,585 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:37,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:38,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:39,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:40,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:40,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:41,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:42,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:43,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:44,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:03:44,063 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:03:44,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:45,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:45,163 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:45,588 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml
+2025-01-06 17:03:46,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:47,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:48,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:49,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:50,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:51,065 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:51,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:52,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:53,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:54,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:55,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:56,065 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:56,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:57,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:58,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:59,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:03:59,063 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:03:59,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:00,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:01,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:01,157 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:02,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:03,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:04,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:05,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:06,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:07,065 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:07,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:08,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:09,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:10,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:11,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:12,065 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:12,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:13,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:13,718 DEBUG   SystemMonitor:2084424 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-06 17:04:13,718 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:04:14,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:04:14,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:04:14,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:15,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:16,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:17,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:17,197 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:18,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:19,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:20,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:20,597 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:21,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:22,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:22,598 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:22,609 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:23,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:24,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:24,598 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:25,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:26,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:27,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:28,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:28,109 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:29,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:04:29,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:04:29,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:30,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:30,602 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:31,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:32,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:32,603 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:33,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:33,686 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:34,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:35,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:36,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:37,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:38,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:39,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:39,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:40,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:41,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:42,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:43,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:43,719 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:04:44,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:04:44,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:04:44,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:44,214 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:44,606 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:45,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:46,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:47,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:48,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:49,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:50,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:50,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:51,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:51,370 DEBUG   SenderThread:2084424 [sender.py:send():379] send: files
+2025-01-06 17:04:51,371 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 with policy now
+2025-01-06 17:04:51,371 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:04:51,372 DEBUG   SenderThread:2084424 [sender.py:send():379] send: history
+2025-01-06 17:04:51,372 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:04:51,372 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:04:51,590 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/bhwker01-media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4
+2025-01-06 17:04:51,607 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json
+2025-01-06 17:04:51,608 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4
+2025-01-06 17:04:51,608 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos
+2025-01-06 17:04:51,608 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media
+2025-01-06 17:04:51,608 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval
+2025-01-06 17:04:51,862 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/6xlmozqq-media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4
+2025-01-06 17:04:52,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:52,608 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:53,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:54,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:55,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:55,825 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:56,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:56,609 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:57,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:58,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:58,610 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:59,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:04:59,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:04:59,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:00,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:01,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:01,154 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:02,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:03,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:04,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:05,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:06,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:07,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:07,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:08,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:09,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:10,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:11,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:12,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:13,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:13,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:13,719 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:05:14,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:05:14,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:05:14,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:15,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:16,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:17,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:18,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:18,214 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:19,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:20,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:21,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:22,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:23,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:24,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:24,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:25,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:26,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:27,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:28,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:29,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:05:29,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:05:29,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:29,138 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:30,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:31,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:32,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:33,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:34,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:35,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:35,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:36,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:37,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:38,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:39,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:40,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:40,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:41,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:42,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:43,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:43,624 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0
+2025-01-06 17:05:43,720 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:05:43,864 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/02f8dbvy-events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0
+2025-01-06 17:05:44,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:05:44,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:05:44,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:44,625 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:05:45,083 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:45,166 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:46,083 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:46,626 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:05:47,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:48,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:48,626 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:05:49,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:50,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:51,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:51,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:52,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:53,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:54,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:55,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:56,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:56,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:57,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:58,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:59,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:05:59,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:05:59,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:00,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:01,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:01,139 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:02,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:03,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:04,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:05,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:06,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:07,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:07,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:08,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:09,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:10,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:11,087 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:12,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:12,087 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:13,087 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:13,724 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:06:14,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:06:14,065 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:06:14,087 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:14,634 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:14,831 DEBUG   SenderThread:2084424 [sender.py:send():379] send: files
+2025-01-06 17:06:14,831 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 with policy now
+2025-01-06 17:06:14,834 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:06:14,834 DEBUG   SenderThread:2084424 [sender.py:send():379] send: history
+2025-01-06 17:06:14,834 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:06:14,835 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:06:15,049 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/wf7exzyr-media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4
+2025-01-06 17:06:15,087 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:15,635 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json
+2025-01-06 17:06:15,635 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4
+2025-01-06 17:06:15,635 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval
+2025-01-06 17:06:15,896 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/1s63098y-media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4
+2025-01-06 17:06:16,090 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:16,635 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:17,091 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:17,850 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:18,092 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:19,090 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:20,094 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:20,636 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:21,117 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:22,092 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:22,637 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:23,033 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:23,094 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:24,097 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:24,638 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:25,093 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:26,102 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:27,095 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:28,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:28,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:29,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:06:29,084 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:06:29,125 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:29,409 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:06:29,409 DEBUG   SenderThread:2084424 [sender.py:send():379] send: files
+2025-01-06 17:06:29,410 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 with policy now
+2025-01-06 17:06:29,410 DEBUG   SenderThread:2084424 [sender.py:send():379] send: history
+2025-01-06 17:06:29,410 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:06:29,410 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:06:29,411 DEBUG   SenderThread:2084424 [sender.py:send():379] send: telemetry
+2025-01-06 17:06:29,413 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: server_info
+2025-01-06 17:06:29,413 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: server_info
+2025-01-06 17:06:29,475 DEBUG   SenderThread:2084424 [sender.py:send():379] send: exit
+2025-01-06 17:06:29,475 INFO    SenderThread:2084424 [sender.py:send_exit():586] handling exit code: 0
+2025-01-06 17:06:29,475 INFO    SenderThread:2084424 [sender.py:send_exit():588] handling runtime: 195
+2025-01-06 17:06:29,475 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:06:29,475 INFO    SenderThread:2084424 [sender.py:send_exit():594] send defer
+2025-01-06 17:06:29,476 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:29,476 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-06 17:06:29,476 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:29,476 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-06 17:06:29,476 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 1
+2025-01-06 17:06:29,476 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:29,476 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-06 17:06:29,476 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:29,476 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-06 17:06:29,476 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 2
+2025-01-06 17:06:29,476 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:29,476 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-06 17:06:29,476 INFO    HandlerThread:2084424 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-06 17:06:29,476 DEBUG   SystemMonitor:2084424 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-06 17:06:29,476 INFO    HandlerThread:2084424 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-06 17:06:29,476 DEBUG   SystemMonitor:2084424 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-06 17:06:29,477 INFO    HandlerThread:2084424 [interfaces.py:finish():200] Joined disk monitor
+2025-01-06 17:06:29,529 INFO    HandlerThread:2084424 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-06 17:06:29,530 INFO    HandlerThread:2084424 [interfaces.py:finish():200] Joined memory monitor
+2025-01-06 17:06:29,530 INFO    HandlerThread:2084424 [interfaces.py:finish():200] Joined network monitor
+2025-01-06 17:06:29,530 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:29,530 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-06 17:06:29,530 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 3
+2025-01-06 17:06:29,530 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:06:29,530 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:29,530 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-06 17:06:29,531 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:29,531 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-06 17:06:29,531 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 4
+2025-01-06 17:06:29,531 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:29,531 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-06 17:06:29,580 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/mgks1ocu-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-06 17:06:29,640 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json
+2025-01-06 17:06:29,640 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-06 17:06:29,640 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval
+2025-01-06 17:06:29,889 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/oos8d3cl-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-06 17:06:30,640 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:36,522 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:06:36,522 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:36,522 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-06 17:06:36,523 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:36,524 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-06 17:06:36,524 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 5
+2025-01-06 17:06:36,524 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:06:36,524 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:36,524 DEBUG   SenderThread:2084424 [sender.py:send():379] send: history
+2025-01-06 17:06:36,524 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-06 17:06:36,524 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:06:36,526 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:06:36,526 DEBUG   SenderThread:2084424 [sender.py:send():379] send: summary
+2025-01-06 17:06:36,526 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:06:36,526 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:36,527 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-06 17:06:36,527 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 6
+2025-01-06 17:06:36,527 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:36,527 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-06 17:06:36,527 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:36,527 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-06 17:06:36,530 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:36,647 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 7
+2025-01-06 17:06:36,647 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:36,647 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-06 17:06:36,648 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:36,648 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-06 17:06:36,648 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml
+2025-01-06 17:06:36,648 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json
+2025-01-06 17:06:37,492 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:06:38,432 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 8
+2025-01-06 17:06:38,432 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:06:38,432 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:38,432 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-06 17:06:38,432 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:38,432 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-06 17:06:38,433 INFO    SenderThread:2084424 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-06 17:06:38,435 INFO    SenderThread:2084424 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-06 17:06:38,453 INFO    SenderThread:2084424 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-06 17:06:38,467 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 9
+2025-01-06 17:06:38,467 DEBUG   SenderThread:2084424 [sender.py:send():379] send: artifact
+2025-01-06 17:06:38,468 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:38,468 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-06 17:06:38,507 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:06:38,649 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:39,231 INFO    wandb-upload_2:2084424 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpr7_awawm
+2025-01-06 17:06:39,385 INFO    wandb-upload_1:2084424 [upload_job.py:push():88] Uploaded file /tmp/tmp3u9o03__/wandb-job.json
+2025-01-06 17:06:39,385 INFO    wandb-upload_0:2084424 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppkyoiie5
+2025-01-06 17:06:39,948 INFO    SenderThread:2084424 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTg5Nzc3Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}}
+2025-01-06 17:06:39,948 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:39,948 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-06 17:06:39,948 INFO    SenderThread:2084424 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-06 17:06:40,650 INFO    SenderThread:2084424 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files
+2025-01-06 17:06:40,650 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml config.yaml
+2025-01-06 17:06:40,650 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json wandb-summary.json
+2025-01-06 17:06:40,652 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt requirements.txt
+2025-01-06 17:06:40,652 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json wandb-metadata.json
+2025-01-06 17:06:40,652 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0
+2025-01-06 17:06:40,653 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log output.log
+2025-01-06 17:06:40,653 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch diff.patch
+2025-01-06 17:06:40,656 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-06 17:06:40,656 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-06 17:06:40,656 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4
+2025-01-06 17:06:40,656 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4
+2025-01-06 17:06:40,656 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 10
+2025-01-06 17:06:40,657 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:06:40,657 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:40,657 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-06 17:06:40,657 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:40,657 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-06 17:06:40,657 INFO    SenderThread:2084424 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 17:06:40,830 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt
+2025-01-06 17:06:40,855 INFO    wandb-upload_3:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:40,866 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json
+2025-01-06 17:06:40,878 INFO    wandb-upload_2:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml
+2025-01-06 17:06:41,078 INFO    Thread-11 (_thread_body):2084424 [sender.py:transition_state():614] send defer: 11
+2025-01-06 17:06:41,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:41,079 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-06 17:06:41,079 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:41,079 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-06 17:06:41,079 INFO    SenderThread:2084424 [file_pusher.py:join():175] waiting for file pusher
+2025-01-06 17:06:41,080 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 12
+2025-01-06 17:06:41,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:41,080 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-06 17:06:41,080 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:41,080 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-06 17:06:41,080 INFO    SenderThread:2084424 [file_stream.py:finish():601] file stream finish called
+2025-01-06 17:06:41,286 INFO    SenderThread:2084424 [file_stream.py:finish():605] file stream finish is done
+2025-01-06 17:06:41,286 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 13
+2025-01-06 17:06:41,286 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:41,286 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-06 17:06:41,286 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:41,286 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-06 17:06:41,286 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 14
+2025-01-06 17:06:41,286 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:41,287 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-06 17:06:41,287 DEBUG   SenderThread:2084424 [sender.py:send():379] send: final
+2025-01-06 17:06:41,287 DEBUG   SenderThread:2084424 [sender.py:send():379] send: footer
+2025-01-06 17:06:41,287 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:41,287 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-06 17:06:41,292 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:06:41,292 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:06:41,313 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:41,321 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-06 17:06:41,326 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-06 17:06:41,329 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-06 17:06:41,329 INFO    HandlerThread:2084424 [handler.py:finish():882] shutting down handler
+2025-01-06 17:06:42,292 INFO    WriterThread:2084424 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb
+2025-01-06 17:06:42,292 INFO    SenderThread:2084424 [sender.py:finish():1615] shutting down sender
+2025-01-06 17:06:42,293 INFO    SenderThread:2084424 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 17:06:42,293 INFO    SenderThread:2084424 [file_pusher.py:join():175] waiting for file pusher

3b422grv/wandb/debug.log

@@ -0,0 +1,37 @@
+2025-01-06 17:03:13,341 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-06 17:03:13,341 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Configure stats pid to 2084243
+2025-01-06 17:03:13,341 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-06 17:03:13,341 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/settings
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug.log
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug-internal.log
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:init():569] calling init triggers
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv'}
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:init():619] starting backend
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:init():623] setting up manager
+2025-01-06 17:03:13,346 INFO    MainThread:2084243 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-06 17:03:13,348 INFO    MainThread:2084243 [wandb_init.py:init():631] backend started and connected
+2025-01-06 17:03:13,360 INFO    MainThread:2084243 [wandb_init.py:init():720] updated telemetry
+2025-01-06 17:03:13,365 INFO    MainThread:2084243 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-06 17:03:13,598 INFO    MainThread:2084243 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-06 17:03:13,699 INFO    MainThread:2084243 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-06 17:03:13,699 INFO    MainThread:2084243 [wandb_init.py:init():804] starting run threads in backend
+2025-01-06 17:03:14,063 INFO    MainThread:2084243 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-06 17:03:14,063 INFO    MainThread:2084243 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-06 17:03:14,063 INFO    MainThread:2084243 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-06 17:03:14,063 INFO    MainThread:2084243 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-06 17:03:14,066 INFO    MainThread:2084243 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-06 17:03:14,066 INFO    MainThread:2084243 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/tb, True
+2025-01-06 17:03:20,151 INFO    MainThread:2084243 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-06 17:06:29,401 INFO    MainThread:2084243 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/3b422grv
+2025-01-06 17:06:29,403 INFO    MainThread:2084243 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0
+2025-01-06 17:06:29,407 INFO    MainThread:2084243 [wandb_run.py:_restore():2352] restore
+2025-01-06 17:06:29,408 INFO    MainThread:2084243 [wandb_run.py:_restore():2358] restore done
+2025-01-06 17:06:42,331 INFO    MainThread:2084243 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-06 17:06:42,332 INFO    MainThread:2084243 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-06 17:06:42,380 INFO    MainThread:2084243 [wandb_run.py:_footer_sync_info():3975] logging synced files

3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py

@@ -0,0 +1,605 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+import binomial_cis as bc
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		self.alpha_range = [.01, .025, .05, .1]
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			# get the run id from the final_experiment_dir directory
+			run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir))
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		wandb.finish()
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+
+				# log confidence intervals for success rate
+				k = self.num_episode_successes # number of successes
+				n = self.global_episode # number of trials
+
+				table_columns = []
+				table_data = []
+				for alpha in self.alpha_range:
+					lb = bc.binom_ci(k, n, alpha, 'lb')
+					ub = bc.binom_ci(k, n, alpha, 'ub')
+
+					self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+					self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+			# log confidence intervals for success rate
+			k = self.num_episode_successes # number of successes
+			n = self.global_episode # number of trials
+
+			table_columns = ['success_rate']
+			table_data = [self.num_episode_successes/self.global_episode]
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(k, n, alpha, 'lb')
+				ub = bc.binom_ci(k, n, alpha, 'ub')
+
+				self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode)
+
+				table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}'])
+				table_data.extend([lb, ub])
+
+			table_data = [table_data]
+
+			# seperately log as a table
+			wandb.log({
+				"eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns)
+			})
+
+			# also accumulate eval metrics across previous eval runs 
+			# TODO: change wandb init to resume from an existing run!!!
+			run_filter={
+				"jobType": "eval",
+				"config.wandb_run_id": self.cfg.wandb_run_id, 
+				"summary_metrics.episode": {"$gte": 5},
+				"config.checkpoint_epoch": self.cfg.checkpoint_epoch,
+				"state": "finished",
+				# "config.grasped_obj_name": grasped_obj_name,
+				# "config.left_book_slot": left_book_slot,
+			}
+
+			api = wandb.Api()
+			filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter)
+			total_num_successes = self.num_episode_successes
+			total_num_episodes = self.global_episode
+			list_of_historical_run_ids = []
+			if len(filtered_runs) > 0:
+				for filtered_run in filtered_runs:
+					total_num_successes += filtered_run.summary_metrics['eval/num_success']
+					# total_num_episodes += filtered_run.summary_metrics['episode']
+					total_num_episodes += filtered_run.config['num_eval']
+					list_of_historical_run_ids.append(filtered_run.id)
+
+			wandb.summary['total_num_successes'] = total_num_successes
+			wandb.summary['total_num_episodes'] = total_num_episodes
+			wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes
+				
+			# log the accumulated metrics as a table
+			total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate']
+			total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes]
+			self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes)
+			
+			for alpha in self.alpha_range:
+				lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb')
+				ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub')
+				total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}'])
+				total_table_data.extend([lb, ub])
+				wandb.summary[f'total_success_rate_lb_{alpha}'] = lb
+				wandb.summary[f'total_success_rate_ub_{alpha}'] = ub
+
+				self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes)
+				self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes)
+
+			total_table_data = [total_table_data]
+			wandb.log({
+				'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns)
+			})
+
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml

@@ -0,0 +1,895 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '1002_0'
+wandb_notes:
+  desc: null
+  value: 1002_0_req_1365_0
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+train_pad_after:
+  desc: null
+  value: 4
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value: all
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 5
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1736200993
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 2
+      - 14
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: greece
+left_book_slot:
+  desc: null
+  value: twodim

3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch

@@ -0,0 +1,65 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index d2afe29..3e5ed57 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -133,12 +133,19 @@ load_checkpoint: ${agent.load_checkpoint}
+ 
+ # all books, 10/20 demos per book
+ # RGBD+mask+act history
+-# wandb_run_id: '1002_0' 
++wandb_run_id: '1002_0' 
++# wandb_run_id: '1009_1' # seed 1
++# wandb_run_id: '1022_0' # dataset shuffle seed 1
++
++# RGBD+mask+contact(all ftrs)+act history
++# wandb_run_id: '1003_0' 
++# wandb_run_id: '1013_1' # seed 1
++# wandb_run_id: '1017_0' # dataset shuffle seed 1
+ 
+ # all books, 6/20 demos per book
+ # RGBD+mask+act history
+ # wandb_run_id: '1007_0' 
+-wandb_run_id: '1922_1' # seed 1
++# wandb_run_id: '1922_1' # seed 1
+ # wandb_run_id: '1938_0' # dataset shuffle seed 1
+ 
+ # RGBD+mask+contact(all ftrs)+act history
+@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1
+ # wandb_run_id: '1926_1' # seed 1
+ # wandb_run_id: '1930_0' # dataset shuffle seed 1
+ 
+-
+-
+ # RGBD+mask+contact(contact+DTC)+act history
+ # wandb_run_id: '1745_0'
+ 
++# all books, 4/20 demos per book
++
+ # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 14 demos
+ 
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index c7eadce..e3c7dc1 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -161,8 +161,18 @@ run_id_list = [
+ run_id_list = [
+     # '1922_1',
+     # '1922_2',
+-    '1926_1',
+-    '1930_0',
++    # '1944_0',
++    # '1972_0',
++    # '1976_1',
++    # '1948_0',
++    # '1950_0',
++    # '1954_1',
++    '1002_0',
++    '1009_1',
++    '1022_0',
++    '1003_0',
++    '1013_1',
++    '1017_0',
+ ]
+ 
+ checkpoint_epoch = 12000

3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d381be8fc37a39636946143ce961a0e9726325f87b53c10b796bff7cecf2908b
+size 1203871

3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43ca52fa14eb631b4abc059d2e4d2836c530e82cde9bbebfb4166f6247188f98
+size 1121306

3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log

@@ -0,0 +1,111 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1736201000.152050]: resetting environment
+[INFO] [1736201000.155322]: cleared current plan
+[INFO] [1736201000.156246]: moving to home
+Move to target timeout: 5.000643730163574. Trans error: 0.05895647034049034. Rot error: 0.7431179070435945
+[INFO] [1736201007.959033]: reached home
+[INFO] [1736201007.960956]: reset action history
+[INFO] [1736201010.677988]: environment reset
+Starting episode 0
+[INFO] [1736201010.680199]: resetting environment
+[INFO] [1736201010.683014]: cleared current plan
+[INFO] [1736201010.684005]: moving to home
+[INFO] [1736201011.686442]: reached home
+[INFO] [1736201011.687768]: reset action history
+[INFO] [1736201014.394347]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1736201059.558569]: resetting environment
+[INFO] [1736201059.564067]: cleared current plan
+[INFO] [1736201059.565510]: moving to home
+[INFO] [1736201060.567596]: reached home
+[INFO] [1736201060.568843]: reset action history
+[INFO] [1736201063.276794]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/episode_rosbags/episode_0_2025-01-06-17-04-23.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2084243.log
+started roslaunch server http://158.130.50.37:39253/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [2084787]
+process[print_text-2]: started with pid [2084811]
+process[rosbag_record-3]: started with pid [2084812]
+started bagging!
+For topic gripper_width: timestamp difference is 106144366 for nearest: 1736201068065174994 - target: 1736201067959030628 at idx 65
+For topic gripper_width: timestamp difference is 102435253 for nearest: 1736201070465177439 - target: 1736201070362742186 at idx 97
+For topic gripper_width: timestamp difference is 102624022 for nearest: 1736201070665201746 - target: 1736201070562577724 at idx 99
+For topic gripper_width: timestamp difference is 101684219 for nearest: 1736201083498508579 - target: 1736201083396824360 at idx 249
+[INFO] [1736201090.318902]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1736201091.124820]: Stored episode 1.
+[INFO] [1736201091.125208]: Saving video...
+[INFO] [1736201091.337960]: Video saved!
+Starting episode 1
+[INFO] [1736201091.372020]: resetting environment
+[INFO] [1736201091.376901]: cleared current plan
+[INFO] [1736201091.378017]: moving to home
+[INFO] [1736201094.782643]: reached home
+[INFO] [1736201094.783742]: reset action history
+[INFO] [1736201097.495492]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+Comuting lower confidence bound
+Comuting upper confidence bound
+[INFO] [1736201143.068640]: resetting environment
+[INFO] [1736201143.073874]: cleared current plan
+[INFO] [1736201143.074064]: moving to home
+[INFO] [1736201144.075095]: reached home
+[INFO] [1736201144.075463]: reset action history
+[INFO] [1736201146.784399]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/episode_rosbags/episode_1_2025-01-06-17-05-46.bag
+... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2084243.log
+started roslaunch server http://158.130.50.37:36435/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [2084924]
+process[print_text-5]: started with pid [2084925]
+process[rosbag_record-6]: started with pid [2084926]
+started bagging!
+[INFO] [1736201173.773907]: Storing episode...
+[rosbag_record-6] killing on exit[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1736201174.531079]: Stored episode 2.
+[INFO] [1736201174.531306]: Saving video...
+[INFO] [1736201174.794614]: Video saved!
+Starting episode 2
+[INFO] [1736201174.837507]: resetting environment
+[INFO] [1736201174.849520]: cleared current plan
+[INFO] [1736201174.849756]: moving to home
+[INFO] [1736201178.352310]: reached home
+[INFO] [1736201178.354759]: reset action history
+[INFO] [1736201181.073285]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+program exiting gracefully
+proceeding to start episode!
+[INFO] [1736201189.372176]: Saving video...
+[INFO] [1736201189.382391]: Video saved!

3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt

@@ -0,0 +1,340 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+binomial_cis==0.0.11
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.3.1
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2025-01-06T22:03:13.727505",
+    "startedAt": "2025-01-06T22:03:13.338476",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3375.356375,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 4499.947,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 311.6428985595703
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}

3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "d381be8fc37a39636946143ce961a0e9726325f87b53c10b796bff7cecf2908b", "size": 1203871, "path": "media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4"}, "global_step": 133, "_timestamp": 1736201143.0684545, "_runtime": 149.7197926044464, "_step": 3, "eval/1_eval": {"_type": "video-file", "sha256": "43ca52fa14eb631b4abc059d2e4d2836c530e82cde9bbebfb4166f6247188f98", "size": 1121306, "path": "media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4"}, "eval/video": {"_type": "video-file", "sha256": "25445a41b5bbda0ed7e2e0845eaa5c96373705c5c738fe640c1ea70a07b4176c", "size": 262, "path": "media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4"}, "_wandb": {"runtime": 195}, "eval/num_success": 0.0, "episode": 1.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.9648889303207397, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.9450690150260925, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.8587992787361145, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.07601606845855713}

3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug-internal.log

@@ -0,0 +1,545 @@
+2025-01-06 17:03:13,347 INFO    StreamThr :2084424 [internal.py:wandb_internal():85] W&B internal server running at pid: 2084424, started at: 2025-01-06 17:03:13.347290
+2025-01-06 17:03:13,348 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status
+2025-01-06 17:03:13,360 INFO    WriterThread:2084424 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb
+2025-01-06 17:03:13,360 DEBUG   SenderThread:2084424 [sender.py:send():379] send: header
+2025-01-06 17:03:13,367 DEBUG   SenderThread:2084424 [sender.py:send():379] send: run
+2025-01-06 17:03:13,578 INFO    SenderThread:2084424 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files
+2025-01-06 17:03:13,578 INFO    SenderThread:2084424 [sender.py:_start_run_threads():1188] run started: 3b422grv with start time 1736200993.348662
+2025-01-06 17:03:13,599 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: check_version
+2025-01-06 17:03:13,599 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: check_version
+2025-01-06 17:03:13,703 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: run_start
+2025-01-06 17:03:13,713 DEBUG   HandlerThread:2084424 [system_info.py:__init__():26] System info init
+2025-01-06 17:03:13,714 DEBUG   HandlerThread:2084424 [system_info.py:__init__():41] System info init done
+2025-01-06 17:03:13,714 INFO    HandlerThread:2084424 [system_monitor.py:start():194] Starting system monitor
+2025-01-06 17:03:13,714 INFO    SystemMonitor:2084424 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2025-01-06 17:03:13,714 INFO    HandlerThread:2084424 [system_monitor.py:probe():214] Collecting system info
+2025-01-06 17:03:13,715 INFO    SystemMonitor:2084424 [interfaces.py:start():188] Started cpu monitoring
+2025-01-06 17:03:13,716 INFO    SystemMonitor:2084424 [interfaces.py:start():188] Started disk monitoring
+2025-01-06 17:03:13,716 INFO    SystemMonitor:2084424 [interfaces.py:start():188] Started gpu monitoring
+2025-01-06 17:03:13,717 INFO    SystemMonitor:2084424 [interfaces.py:start():188] Started memory monitoring
+2025-01-06 17:03:13,717 INFO    SystemMonitor:2084424 [interfaces.py:start():188] Started network monitoring
+2025-01-06 17:03:13,727 DEBUG   HandlerThread:2084424 [system_info.py:probe():152] Probing system
+2025-01-06 17:03:13,728 DEBUG   HandlerThread:2084424 [system_info.py:_probe_git():137] Probing git
+2025-01-06 17:03:13,733 DEBUG   HandlerThread:2084424 [system_info.py:_probe_git():145] Probing git done
+2025-01-06 17:03:13,733 DEBUG   HandlerThread:2084424 [system_info.py:probe():200] Probing system done
+2025-01-06 17:03:13,733 DEBUG   HandlerThread:2084424 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:03:13.727505', 'startedAt': '2025-01-06T22:03:13.338476', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3375.356375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.947, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 311.6428985595703}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2025-01-06 17:03:13,733 INFO    HandlerThread:2084424 [system_monitor.py:probe():224] Finished collecting system info
+2025-01-06 17:03:13,733 INFO    HandlerThread:2084424 [system_monitor.py:probe():227] Publishing system info
+2025-01-06 17:03:13,733 DEBUG   HandlerThread:2084424 [system_info.py:_save_code():44] Saving code
+2025-01-06 17:03:13,735 DEBUG   HandlerThread:2084424 [system_info.py:_save_code():67] Saving code done
+2025-01-06 17:03:13,735 DEBUG   HandlerThread:2084424 [system_info.py:_save_patches():84] Saving git patches
+2025-01-06 17:03:13,796 DEBUG   HandlerThread:2084424 [system_info.py:_save_patches():126] Saving git patches done
+2025-01-06 17:03:13,797 INFO    HandlerThread:2084424 [system_monitor.py:probe():229] Finished publishing system info
+2025-01-06 17:03:13,800 DEBUG   SenderThread:2084424 [sender.py:send():379] send: files
+2025-01-06 17:03:13,800 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2025-01-06 17:03:13,800 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2025-01-06 17:03:13,800 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2025-01-06 17:03:14,054 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/v4bqqb1x-wandb-metadata.json
+2025-01-06 17:03:14,062 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: python_packages
+2025-01-06 17:03:14,063 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: python_packages
+2025-01-06 17:03:14,064 DEBUG   SenderThread:2084424 [sender.py:send():379] send: telemetry
+2025-01-06 17:03:14,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:03:14,065 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:03:14,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:14,066 INFO    HandlerThread:2084424 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/tb"
+  save: true
+}
+control {
+  relay_id: "b0cd36502b184c79a585690d8b1a1e21"
+}
+_info {
+  stream_id: "3b422grv"
+}
+
+2025-01-06 17:03:14,066 DEBUG   HandlerThread:2084424 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2025-01-06 17:03:14,071 INFO    wandb-upload_2:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/axgmigee-diff.patch
+2025-01-06 17:03:14,075 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/rp2vzrhc-code/FISH/eval_robot.py
+2025-01-06 17:03:14,219 DEBUG   SenderThread:2084424 [sender.py:send():379] send: tbrecord
+2025-01-06 17:03:14,219 DEBUG   SenderThread:2084424 [sender.py:send():379] send: files
+2025-01-06 17:03:14,219 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 with policy live
+2025-01-06 17:03:14,395 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/dk5st5le-events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0
+2025-01-06 17:03:14,580 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py
+2025-01-06 17:03:14,580 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:14,580 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0
+2025-01-06 17:03:14,580 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt
+2025-01-06 17:03:14,581 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch
+2025-01-06 17:03:14,581 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json
+2025-01-06 17:03:14,581 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH
+2025-01-06 17:03:14,581 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code
+2025-01-06 17:03:15,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:16,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:16,579 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:17,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:18,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:19,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:19,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:20,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:20,151 DEBUG   SenderThread:2084424 [sender.py:send():379] send: config
+2025-01-06 17:03:21,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:22,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:22,581 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:23,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:24,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:24,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:25,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:26,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:26,582 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:27,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:28,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:29,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:03:29,063 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:03:29,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:29,205 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:30,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:30,583 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:31,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:32,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:32,584 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:33,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:34,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:34,394 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:35,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:36,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:36,585 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:03:37,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:38,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:39,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:40,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:40,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:41,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:42,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:43,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:44,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:03:44,063 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:03:44,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:45,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:45,163 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:45,588 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml
+2025-01-06 17:03:46,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:47,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:48,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:49,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:50,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:51,065 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:51,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:52,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:53,068 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:54,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:55,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:56,065 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:03:56,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:57,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:58,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:03:59,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:03:59,063 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:03:59,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:00,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:01,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:01,157 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:02,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:03,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:04,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:05,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:06,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:07,065 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:07,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:08,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:09,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:10,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:11,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:12,065 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:12,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:13,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:13,718 DEBUG   SystemMonitor:2084424 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2025-01-06 17:04:13,718 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:04:14,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:04:14,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:04:14,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:15,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:16,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:17,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:17,197 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:18,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:19,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:20,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:20,597 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:21,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:22,072 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:22,598 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:22,609 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:23,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:24,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:24,598 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:25,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:26,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:27,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:28,066 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:28,109 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:29,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:04:29,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:04:29,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:30,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:30,602 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:31,073 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:32,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:32,603 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:33,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:33,686 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:34,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:35,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:36,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:37,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:38,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:39,067 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:39,074 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:40,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:41,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:42,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:43,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:43,719 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:04:44,063 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:04:44,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:04:44,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:44,214 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:44,606 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:45,075 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:46,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:47,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:48,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:49,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:50,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:50,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:51,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:51,370 DEBUG   SenderThread:2084424 [sender.py:send():379] send: files
+2025-01-06 17:04:51,371 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 with policy now
+2025-01-06 17:04:51,371 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:04:51,372 DEBUG   SenderThread:2084424 [sender.py:send():379] send: history
+2025-01-06 17:04:51,372 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:04:51,372 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:04:51,590 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/bhwker01-media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4
+2025-01-06 17:04:51,607 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json
+2025-01-06 17:04:51,608 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4
+2025-01-06 17:04:51,608 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos
+2025-01-06 17:04:51,608 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media
+2025-01-06 17:04:51,608 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval
+2025-01-06 17:04:51,862 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/6xlmozqq-media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4
+2025-01-06 17:04:52,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:52,608 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:53,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:54,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:55,076 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:55,825 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:04:56,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:56,609 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:57,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:58,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:04:58,610 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:04:59,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:04:59,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:04:59,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:00,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:01,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:01,154 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:02,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:03,077 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:04,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:05,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:06,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:07,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:07,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:08,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:09,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:10,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:11,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:12,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:13,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:13,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:13,719 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:05:14,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:05:14,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:05:14,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:15,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:16,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:17,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:18,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:18,214 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:19,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:20,079 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:21,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:22,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:23,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:24,069 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:24,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:25,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:26,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:27,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:28,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:29,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:05:29,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:05:29,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:29,138 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:30,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:31,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:32,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:33,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:34,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:35,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:35,081 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:36,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:37,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:38,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:39,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:40,070 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:40,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:41,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:42,082 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:43,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:43,624 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0
+2025-01-06 17:05:43,720 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:05:43,864 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/02f8dbvy-events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0
+2025-01-06 17:05:44,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:05:44,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:05:44,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:44,625 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:05:45,083 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:45,166 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:46,083 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:46,626 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:05:47,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:48,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:48,626 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:05:49,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:50,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:51,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:51,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:52,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:53,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:54,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:55,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:56,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:05:56,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:57,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:58,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:05:59,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:05:59,064 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:05:59,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:00,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:01,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:01,139 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:02,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:03,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:04,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:05,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:06,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:07,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:07,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:08,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:09,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:10,086 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:11,087 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:12,071 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:12,087 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:13,087 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:13,724 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:06:14,064 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:06:14,065 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:06:14,087 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:14,634 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:14,831 DEBUG   SenderThread:2084424 [sender.py:send():379] send: files
+2025-01-06 17:06:14,831 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 with policy now
+2025-01-06 17:06:14,834 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:06:14,834 DEBUG   SenderThread:2084424 [sender.py:send():379] send: history
+2025-01-06 17:06:14,834 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:06:14,835 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:06:15,049 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/wf7exzyr-media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4
+2025-01-06 17:06:15,087 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:15,635 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json
+2025-01-06 17:06:15,635 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4
+2025-01-06 17:06:15,635 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval
+2025-01-06 17:06:15,896 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/1s63098y-media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4
+2025-01-06 17:06:16,090 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:16,635 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:17,091 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:17,850 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:18,092 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:19,090 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:20,094 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:20,636 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:21,117 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:22,092 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:22,637 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:23,033 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:23,094 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:24,097 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:24,638 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:25,093 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:26,102 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:27,095 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:28,085 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:28,105 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:29,084 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: stop_status
+2025-01-06 17:06:29,084 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: stop_status
+2025-01-06 17:06:29,125 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:29,409 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: partial_history
+2025-01-06 17:06:29,409 DEBUG   SenderThread:2084424 [sender.py:send():379] send: files
+2025-01-06 17:06:29,410 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 with policy now
+2025-01-06 17:06:29,410 DEBUG   SenderThread:2084424 [sender.py:send():379] send: history
+2025-01-06 17:06:29,410 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:06:29,410 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:06:29,411 DEBUG   SenderThread:2084424 [sender.py:send():379] send: telemetry
+2025-01-06 17:06:29,413 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: server_info
+2025-01-06 17:06:29,413 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: server_info
+2025-01-06 17:06:29,475 DEBUG   SenderThread:2084424 [sender.py:send():379] send: exit
+2025-01-06 17:06:29,475 INFO    SenderThread:2084424 [sender.py:send_exit():586] handling exit code: 0
+2025-01-06 17:06:29,475 INFO    SenderThread:2084424 [sender.py:send_exit():588] handling runtime: 195
+2025-01-06 17:06:29,475 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:06:29,475 INFO    SenderThread:2084424 [sender.py:send_exit():594] send defer
+2025-01-06 17:06:29,476 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:29,476 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 0
+2025-01-06 17:06:29,476 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:29,476 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 0
+2025-01-06 17:06:29,476 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 1
+2025-01-06 17:06:29,476 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:29,476 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 1
+2025-01-06 17:06:29,476 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:29,476 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 1
+2025-01-06 17:06:29,476 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 2
+2025-01-06 17:06:29,476 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:29,476 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 2
+2025-01-06 17:06:29,476 INFO    HandlerThread:2084424 [system_monitor.py:finish():203] Stopping system monitor
+2025-01-06 17:06:29,476 DEBUG   SystemMonitor:2084424 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2025-01-06 17:06:29,476 INFO    HandlerThread:2084424 [interfaces.py:finish():200] Joined cpu monitor
+2025-01-06 17:06:29,476 DEBUG   SystemMonitor:2084424 [system_monitor.py:_start():183] Publishing last batch of metrics
+2025-01-06 17:06:29,477 INFO    HandlerThread:2084424 [interfaces.py:finish():200] Joined disk monitor
+2025-01-06 17:06:29,529 INFO    HandlerThread:2084424 [interfaces.py:finish():200] Joined gpu monitor
+2025-01-06 17:06:29,530 INFO    HandlerThread:2084424 [interfaces.py:finish():200] Joined memory monitor
+2025-01-06 17:06:29,530 INFO    HandlerThread:2084424 [interfaces.py:finish():200] Joined network monitor
+2025-01-06 17:06:29,530 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:29,530 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 2
+2025-01-06 17:06:29,530 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 3
+2025-01-06 17:06:29,530 DEBUG   SenderThread:2084424 [sender.py:send():379] send: stats
+2025-01-06 17:06:29,530 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:29,530 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 3
+2025-01-06 17:06:29,531 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:29,531 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 3
+2025-01-06 17:06:29,531 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 4
+2025-01-06 17:06:29,531 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:29,531 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 4
+2025-01-06 17:06:29,580 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/mgks1ocu-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-06 17:06:29,640 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json
+2025-01-06 17:06:29,640 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-06 17:06:29,640 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval
+2025-01-06 17:06:29,889 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /tmp/tmpd8nomdi5wandb/oos8d3cl-media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-06 17:06:30,640 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:36,522 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:06:36,522 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:36,522 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: keepalive
+2025-01-06 17:06:36,523 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:36,524 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 4
+2025-01-06 17:06:36,524 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 5
+2025-01-06 17:06:36,524 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:06:36,524 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:36,524 DEBUG   SenderThread:2084424 [sender.py:send():379] send: history
+2025-01-06 17:06:36,524 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 5
+2025-01-06 17:06:36,524 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: summary_record
+2025-01-06 17:06:36,526 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:06:36,526 DEBUG   SenderThread:2084424 [sender.py:send():379] send: summary
+2025-01-06 17:06:36,526 INFO    SenderThread:2084424 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2025-01-06 17:06:36,526 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:36,527 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 5
+2025-01-06 17:06:36,527 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 6
+2025-01-06 17:06:36,527 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:36,527 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 6
+2025-01-06 17:06:36,527 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:36,527 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 6
+2025-01-06 17:06:36,530 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: status_report
+2025-01-06 17:06:36,647 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 7
+2025-01-06 17:06:36,647 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:36,647 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 7
+2025-01-06 17:06:36,648 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:36,648 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 7
+2025-01-06 17:06:36,648 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml
+2025-01-06 17:06:36,648 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json
+2025-01-06 17:06:37,492 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:06:38,432 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 8
+2025-01-06 17:06:38,432 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:06:38,432 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:38,432 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 8
+2025-01-06 17:06:38,432 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:38,432 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 8
+2025-01-06 17:06:38,433 INFO    SenderThread:2084424 [job_builder.py:build():440] Attempting to build job artifact
+2025-01-06 17:06:38,435 INFO    SenderThread:2084424 [job_builder.py:_get_source_type():569] is repo sourced job
+2025-01-06 17:06:38,453 INFO    SenderThread:2084424 [job_builder.py:build():545] adding wandb-job metadata file
+2025-01-06 17:06:38,467 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 9
+2025-01-06 17:06:38,467 DEBUG   SenderThread:2084424 [sender.py:send():379] send: artifact
+2025-01-06 17:06:38,468 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:38,468 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 9
+2025-01-06 17:06:38,507 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:06:38,649 INFO    Thread-12 :2084424 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:39,231 INFO    wandb-upload_2:2084424 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpr7_awawm
+2025-01-06 17:06:39,385 INFO    wandb-upload_1:2084424 [upload_job.py:push():88] Uploaded file /tmp/tmp3u9o03__/wandb-job.json
+2025-01-06 17:06:39,385 INFO    wandb-upload_0:2084424 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppkyoiie5
+2025-01-06 17:06:39,948 INFO    SenderThread:2084424 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTg5Nzc3Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxOTE5MjI0Ng==', 'versionIndex': 420}}}
+2025-01-06 17:06:39,948 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:39,948 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 9
+2025-01-06 17:06:39,948 INFO    SenderThread:2084424 [dir_watcher.py:finish():358] shutting down directory watcher
+2025-01-06 17:06:40,650 INFO    SenderThread:2084424 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files
+2025-01-06 17:06:40,650 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml config.yaml
+2025-01-06 17:06:40,650 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json wandb-summary.json
+2025-01-06 17:06:40,652 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt requirements.txt
+2025-01-06 17:06:40,652 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-metadata.json wandb-metadata.json
+2025-01-06 17:06:40,652 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0 events.out.tfevents.1736200994.leonmkim-ROG-Strix-G15CS-G15CS.2084243.0
+2025-01-06 17:06:40,653 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log output.log
+2025-01-06 17:06:40,653 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/diff.patch diff.patch
+2025-01-06 17:06:40,656 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2025-01-06 17:06:40,656 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_2_25445a41b5bbda0ed7e2.mp4
+2025-01-06 17:06:40,656 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4 media/videos/eval/0_eval_0_d381be8fc37a39636946.mp4
+2025-01-06 17:06:40,656 INFO    SenderThread:2084424 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4 media/videos/eval/1_eval_1_43ca52fa14eb631b4abc.mp4
+2025-01-06 17:06:40,656 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 10
+2025-01-06 17:06:40,657 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:06:40,657 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:40,657 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 10
+2025-01-06 17:06:40,657 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:40,657 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 10
+2025-01-06 17:06:40,657 INFO    SenderThread:2084424 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 17:06:40,830 INFO    wandb-upload_1:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/requirements.txt
+2025-01-06 17:06:40,855 INFO    wandb-upload_3:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/output.log
+2025-01-06 17:06:40,866 INFO    wandb-upload_0:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/wandb-summary.json
+2025-01-06 17:06:40,878 INFO    wandb-upload_2:2084424 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/files/config.yaml
+2025-01-06 17:06:41,078 INFO    Thread-11 (_thread_body):2084424 [sender.py:transition_state():614] send defer: 11
+2025-01-06 17:06:41,078 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:41,079 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 11
+2025-01-06 17:06:41,079 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:41,079 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 11
+2025-01-06 17:06:41,079 INFO    SenderThread:2084424 [file_pusher.py:join():175] waiting for file pusher
+2025-01-06 17:06:41,080 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 12
+2025-01-06 17:06:41,080 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:41,080 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 12
+2025-01-06 17:06:41,080 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:41,080 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 12
+2025-01-06 17:06:41,080 INFO    SenderThread:2084424 [file_stream.py:finish():601] file stream finish called
+2025-01-06 17:06:41,286 INFO    SenderThread:2084424 [file_stream.py:finish():605] file stream finish is done
+2025-01-06 17:06:41,286 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 13
+2025-01-06 17:06:41,286 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:41,286 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 13
+2025-01-06 17:06:41,286 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:41,286 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 13
+2025-01-06 17:06:41,286 INFO    SenderThread:2084424 [sender.py:transition_state():614] send defer: 14
+2025-01-06 17:06:41,286 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: defer
+2025-01-06 17:06:41,287 INFO    HandlerThread:2084424 [handler.py:handle_request_defer():184] handle defer: 14
+2025-01-06 17:06:41,287 DEBUG   SenderThread:2084424 [sender.py:send():379] send: final
+2025-01-06 17:06:41,287 DEBUG   SenderThread:2084424 [sender.py:send():379] send: footer
+2025-01-06 17:06:41,287 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: defer
+2025-01-06 17:06:41,287 INFO    SenderThread:2084424 [sender.py:send_request_defer():610] handle sender defer: 14
+2025-01-06 17:06:41,292 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: poll_exit
+2025-01-06 17:06:41,292 DEBUG   SenderThread:2084424 [sender.py:send_request():406] send_request: poll_exit
+2025-01-06 17:06:41,313 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: internal_messages
+2025-01-06 17:06:41,321 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: get_summary
+2025-01-06 17:06:41,326 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: sampled_history
+2025-01-06 17:06:41,329 DEBUG   HandlerThread:2084424 [handler.py:handle_request():158] handle_request: shutdown
+2025-01-06 17:06:41,329 INFO    HandlerThread:2084424 [handler.py:finish():882] shutting down handler
+2025-01-06 17:06:42,292 INFO    WriterThread:2084424 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/run-3b422grv.wandb
+2025-01-06 17:06:42,292 INFO    SenderThread:2084424 [sender.py:finish():1615] shutting down sender
+2025-01-06 17:06:42,293 INFO    SenderThread:2084424 [file_pusher.py:finish():169] shutting down file pusher
+2025-01-06 17:06:42,293 INFO    SenderThread:2084424 [file_pusher.py:join():175] waiting for file pusher

3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug.log

@@ -0,0 +1,37 @@
+2025-01-06 17:03:13,341 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2025-01-06 17:03:13,341 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Configure stats pid to 2084243
+2025-01-06 17:03:13,341 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2025-01-06 17:03:13,341 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/settings
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_setup.py:_flush():76] Applying login settings: {}
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug.log
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/wandb/run-20250106_170313-3b422grv/logs/debug-internal.log
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:init():569] calling init triggers
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv'}
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:init():619] starting backend
+2025-01-06 17:03:13,342 INFO    MainThread:2084243 [wandb_init.py:init():623] setting up manager
+2025-01-06 17:03:13,346 INFO    MainThread:2084243 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-01-06 17:03:13,348 INFO    MainThread:2084243 [wandb_init.py:init():631] backend started and connected
+2025-01-06 17:03:13,360 INFO    MainThread:2084243 [wandb_init.py:init():720] updated telemetry
+2025-01-06 17:03:13,365 INFO    MainThread:2084243 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2025-01-06 17:03:13,598 INFO    MainThread:2084243 [wandb_run.py:_on_init():2435] communicating current version
+2025-01-06 17:03:13,699 INFO    MainThread:2084243 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2025-01-06 17:03:13,699 INFO    MainThread:2084243 [wandb_init.py:init():804] starting run threads in backend
+2025-01-06 17:03:14,063 INFO    MainThread:2084243 [wandb_run.py:_console_start():2413] atexit reg
+2025-01-06 17:03:14,063 INFO    MainThread:2084243 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2025-01-06 17:03:14,063 INFO    MainThread:2084243 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2025-01-06 17:03:14,063 INFO    MainThread:2084243 [wandb_run.py:_redirect():2345] Redirects installed.
+2025-01-06 17:03:14,066 INFO    MainThread:2084243 [wandb_init.py:init():847] run started, returning control to user process
+2025-01-06 17:03:14,066 INFO    MainThread:2084243 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/3b422grv/tb, True
+2025-01-06 17:03:20,151 INFO    MainThread:2084243 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2025-01-06 17:06:29,401 INFO    MainThread:2084243 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/3b422grv
+2025-01-06 17:06:29,403 INFO    MainThread:2084243 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0
+2025-01-06 17:06:29,407 INFO    MainThread:2084243 [wandb_run.py:_restore():2352] restore
+2025-01-06 17:06:29,408 INFO    MainThread:2084243 [wandb_run.py:_restore():2358] restore done
+2025-01-06 17:06:42,331 INFO    MainThread:2084243 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2025-01-06 17:06:42,332 INFO    MainThread:2084243 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2025-01-06 17:06:42,380 INFO    MainThread:2084243 [wandb_run.py:_footer_sync_info():3975] logging synced files

b3d2fbi3/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

b3d2fbi3/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

b3d2fbi3/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

b3d2fbi3/eval_robot.log

@@ -0,0 +1,12 @@
+[2025-01-07 21:43:26,892][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-07 21:43:26,895][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

df8ok4xt/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1002_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

df8ok4xt/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/df8ok4xt
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

df8ok4xt/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

df8ok4xt/eval_policy.log

@@ -0,0 +1,15 @@
+[2025-01-06 17:03:03,813][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2025-01-06 17:03:03,816][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2025-01-06 17:03:06,613][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

j5vura5m/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '1002_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

j5vura5m/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/j5vura5m
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

j5vura5m/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

j5vura5m/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

j5vura5m/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

j5vura5m/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

j5vura5m/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

j5vura5m/episode_rosbags/episode_2_2025-01-07-17-20-22.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:819641a8054b5398caffea3774b3f9e461816c9bf29a77c9ec32e30b6b64404a
+size 1409533772