Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,29 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb filter=lfs diff=lfs merge=lfs -text
+212118/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 filter=lfs diff=lfs merge=lfs -text
+212118/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 filter=lfs diff=lfs merge=lfs -text
+212118/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 filter=lfs diff=lfs merge=lfs -text
+215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 filter=lfs diff=lfs merge=lfs -text
+212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 filter=lfs diff=lfs merge=lfs -text
+215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 filter=lfs diff=lfs merge=lfs -text
+215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 filter=lfs diff=lfs merge=lfs -text
+215240/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 filter=lfs diff=lfs merge=lfs -text
+215240/wandb/run-20241217_215246-g6nsr00w/run-g6nsr00w.wandb filter=lfs diff=lfs merge=lfs -text
+215240/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+215240/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+215240/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+215240/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+212118/episode_rosbags/episode_1_2024-12-17-21-22-40.bag filter=lfs diff=lfs merge=lfs -text
+212118/episode_rosbags/episode_0_2024-12-17-21-21-52.bag filter=lfs diff=lfs merge=lfs -text
+215240/episode_rosbags/episode_3_2024-12-17-21-55-35.bag filter=lfs diff=lfs merge=lfs -text
+215240/episode_rosbags/episode_0_2024-12-17-21-53-24.bag filter=lfs diff=lfs merge=lfs -text
+215240/episode_rosbags/episode_2_2024-12-17-21-54-50.bag filter=lfs diff=lfs merge=lfs -text
+215240/episode_rosbags/episode_1_2024-12-17-21-54-05.bag filter=lfs diff=lfs merge=lfs -text
+212118/episode_rosbags/episode_2_2024-12-17-21-23-24.bag filter=lfs diff=lfs merge=lfs -text
+215240/episode_rosbags/episode_4_2024-12-17-21-56-22.bag filter=lfs diff=lfs merge=lfs -text

212117/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '523_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

212117/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212117
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

212117/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

212117/eval_policy.log

@@ -0,0 +1,15 @@
+[2024-12-17 21:21:17,349][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 21:21:17,352][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-17 21:21:19,913][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

212118/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '523_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

212118/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

212118/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

212118/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

212118/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

212118/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

212118/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

212118/episode_rosbags/episode_0_2024-12-17-21-21-52.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39f43013b9eb9297f3fba753c7675143717f98c589ae81ac215d1d09aac5c6b9
+size 977351999

212118/episode_rosbags/episode_1_2024-12-17-21-22-40.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a18725042661a4db0597571d90cda790d55ec986b7265721a0e69644ae2239e3
+size 953326085

212118/episode_rosbags/episode_2_2024-12-17-21-23-24.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ce68c81d6da0c193b4e1c2cdc12989a0efcac454b23dc612fcae86932e72a50
+size 1416813567

212118/eval_robot.log

@@ -0,0 +1,12 @@
+[2024-12-17 21:21:18,583][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 21:21:18,586][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

212118/eval_video/0_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7012418043e20ba472ca099eb2fd4bae9cf951ff5f200bf3b8b28bef7c455f17
+size 860246

212118/eval_video/1_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a30824235f8d3481acdd75328ce66a4db21dec3ebd5b875ba7e3c527508d0f3
+size 854722

212118/eval_video/2_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe502a2ffcc54aa7653539234f578d5a1a01cd0bf111c0cb8107c59433150318
+size 1236066

212118/tb/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfdd281d85e27f83b5cfc0d25f514c1de66959b75befbd84590bc5f3ed7e29b5
+size 490

212118/wandb/debug-internal.log

@@ -0,0 +1,651 @@
+2024-12-17 21:21:24,431 INFO    StreamThr :76461 [internal.py:wandb_internal():85] W&B internal server running at pid: 76461, started at: 2024-12-17 21:21:24.431534
+2024-12-17 21:21:24,433 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status
+2024-12-17 21:21:24,444 INFO    WriterThread:76461 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb
+2024-12-17 21:21:24,444 DEBUG   SenderThread:76461 [sender.py:send():379] send: header
+2024-12-17 21:21:24,453 DEBUG   SenderThread:76461 [sender.py:send():379] send: run
+2024-12-17 21:21:24,691 INFO    SenderThread:76461 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files
+2024-12-17 21:21:24,691 INFO    SenderThread:76461 [sender.py:_start_run_threads():1188] run started: 78r9r705 with start time 1734488484.432173
+2024-12-17 21:21:24,705 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 21:21:24,706 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: check_version
+2024-12-17 21:21:24,787 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 21:21:24,792 DEBUG   HandlerThread:76461 [system_info.py:__init__():26] System info init
+2024-12-17 21:21:24,792 DEBUG   HandlerThread:76461 [system_info.py:__init__():41] System info init done
+2024-12-17 21:21:24,792 INFO    HandlerThread:76461 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 21:21:24,792 INFO    SystemMonitor:76461 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 21:21:24,792 INFO    HandlerThread:76461 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 21:21:24,793 INFO    SystemMonitor:76461 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 21:21:24,793 INFO    SystemMonitor:76461 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 21:21:24,793 INFO    SystemMonitor:76461 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 21:21:24,794 INFO    SystemMonitor:76461 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 21:21:24,794 INFO    SystemMonitor:76461 [interfaces.py:start():188] Started network monitoring
+2024-12-17 21:21:24,806 DEBUG   HandlerThread:76461 [system_info.py:probe():152] Probing system
+2024-12-17 21:21:24,807 DEBUG   HandlerThread:76461 [system_info.py:_probe_git():137] Probing git
+2024-12-17 21:21:24,812 DEBUG   HandlerThread:76461 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 21:21:24,812 DEBUG   HandlerThread:76461 [system_info.py:probe():200] Probing system done
+2024-12-17 21:21:24,812 DEBUG   HandlerThread:76461 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T02:21:24.806252', 'startedAt': '2024-12-18T02:21:24.423927', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3375.3751250000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4501.58, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.929, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 389.73836517333984}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 21:21:24,812 INFO    HandlerThread:76461 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 21:21:24,812 INFO    HandlerThread:76461 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 21:21:24,812 DEBUG   HandlerThread:76461 [system_info.py:_save_code():44] Saving code
+2024-12-17 21:21:24,814 DEBUG   HandlerThread:76461 [system_info.py:_save_code():67] Saving code done
+2024-12-17 21:21:24,814 DEBUG   HandlerThread:76461 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 21:21:24,883 DEBUG   HandlerThread:76461 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 21:21:24,883 INFO    HandlerThread:76461 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 21:21:24,886 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:21:24,886 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 21:21:24,887 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 21:21:24,887 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 21:21:25,126 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 21:21:25,126 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 21:21:25,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:25,128 DEBUG   SenderThread:76461 [sender.py:send():379] send: telemetry
+2024-12-17 21:21:25,133 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/zf5gfm67-diff.patch
+2024-12-17 21:21:25,157 INFO    wandb-upload_1:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/bk0z0q6f-code/FISH/eval_robot.py
+2024-12-17 21:21:25,159 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/o7jx2utn-wandb-metadata.json
+2024-12-17 21:21:25,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:21:25,169 INFO    HandlerThread:76461 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/tb"
+  save: true
+}
+control {
+  relay_id: "52ec4a1767a3429cb6d1b8e86029eaaf"
+}
+_info {
+  stream_id: "78r9r705"
+}
+
+2024-12-17 21:21:25,169 DEBUG   HandlerThread:76461 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 21:21:25,170 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:21:25,277 DEBUG   SenderThread:76461 [sender.py:send():379] send: tbrecord
+2024-12-17 21:21:25,278 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:21:25,278 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 with policy live
+2024-12-17 21:21:25,454 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/lobsc1b3-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:25,694 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH
+2024-12-17 21:21:25,694 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code
+2024-12-17 21:21:26,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:27,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:27,692 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:28,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:29,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:30,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:30,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:31,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:31,696 DEBUG   SenderThread:76461 [sender.py:send():379] send: config
+2024-12-17 21:21:32,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:33,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:33,694 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:34,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:35,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:35,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:36,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:37,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:37,695 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:38,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:39,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:40,126 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:21:40,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:21:40,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:40,281 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:41,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:41,696 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:42,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:43,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:44,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:45,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:46,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:46,130 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:47,130 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:48,130 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:49,130 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:50,130 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:51,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:51,699 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:52,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:52,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:53,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:53,699 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:54,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:55,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:21:55,129 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:21:55,173 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:55,700 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:56,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:57,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:57,282 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:57,701 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml
+2024-12-17 21:21:57,701 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:58,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:59,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:59,701 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:00,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:01,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:01,702 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:02,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:02,562 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:03,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:03,702 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:04,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:05,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:05,703 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:06,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:07,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:07,703 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:07,736 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:08,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:09,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:09,704 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:10,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:22:10,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:22:10,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:11,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:11,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:12,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:12,599 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:22:12,599 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 with policy now
+2024-12-17 21:22:12,599 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:22:12,600 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:22:12,600 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:22:12,601 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:22:12,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4
+2024-12-17 21:22:12,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:22:12,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval
+2024-12-17 21:22:12,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media
+2024-12-17 21:22:12,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos
+2024-12-17 21:22:12,849 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/0wiijtor-media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4
+2024-12-17 21:22:13,108 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/2ltzk93y-media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4
+2024-12-17 21:22:13,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:13,611 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:13,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:14,138 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:15,134 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:16,134 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:17,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:17,707 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:18,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:18,625 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:19,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:19,707 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:20,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:21,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:22,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:23,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:24,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:24,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:24,794 DEBUG   SystemMonitor:76461 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 21:22:24,795 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:22:25,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:22:25,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:22:25,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:26,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:27,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:28,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:29,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:29,217 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:30,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:31,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:32,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:33,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:34,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:35,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:35,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:36,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:37,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:37,713 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:22:37,978 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/lzwjgswx-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:22:38,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:39,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:39,714 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:40,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:22:40,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:22:40,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:40,228 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:41,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:41,715 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:42,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:43,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:43,715 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:44,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:45,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:45,252 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:45,716 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:46,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:47,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:47,717 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:48,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:49,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:49,718 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:50,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:50,256 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:51,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:51,718 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:52,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:53,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:53,719 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:54,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:54,796 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:22:55,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:22:55,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:22:55,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:55,267 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:55,720 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:56,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:57,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:57,720 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:58,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:59,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:59,721 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:00,142 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:00,373 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:00,417 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:23:00,417 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:23:00,418 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 with policy now
+2024-12-17 21:23:00,418 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:23:00,418 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:23:00,418 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:23:00,673 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/4e5seuc5-media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4
+2024-12-17 21:23:00,721 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:23:00,722 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4
+2024-12-17 21:23:00,722 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval
+2024-12-17 21:23:00,967 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/akortq61-media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4
+2024-12-17 21:23:01,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:01,722 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:02,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:03,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:04,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:05,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:05,723 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:06,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:06,160 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:07,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:07,724 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:08,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:09,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:10,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:23:10,131 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:23:10,173 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:11,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:11,232 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:12,146 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:13,146 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:14,146 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:15,146 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:16,146 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:17,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:17,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:18,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:19,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:20,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:21,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:21,571 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:23:21,571 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:23:21,571 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:23:21,729 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:23:21,729 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:23:21,729 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:22,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:23,029 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:23,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:23,729 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:24,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:24,796 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:23:25,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:23:25,128 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:23:25,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:25,730 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:26,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:27,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:27,731 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:28,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:28,194 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:29,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:29,731 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:30,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:31,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:31,732 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:32,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:33,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:33,393 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:33,733 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:34,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:35,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:35,734 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:36,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:37,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:37,734 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:38,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:38,405 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:39,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:39,735 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:40,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:23:40,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:23:40,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:41,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:41,736 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:42,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:43,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:43,409 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:43,737 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:44,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:45,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:45,737 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:46,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:47,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:47,738 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:48,151 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:48,460 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:49,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:49,739 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:50,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:51,151 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:51,740 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:52,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:52,900 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:23:52,900 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 with policy now
+2024-12-17 21:23:52,900 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:23:52,901 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:23:52,901 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:23:52,901 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:23:53,155 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/evzpq5tf-media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4
+2024-12-17 21:23:53,155 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:53,741 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:23:53,741 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:53,741 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4
+2024-12-17 21:23:53,741 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval
+2024-12-17 21:23:53,920 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:54,016 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/jsf3d998-media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4
+2024-12-17 21:23:54,163 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:54,797 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:23:55,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:23:55,133 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:23:55,177 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:56,161 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:57,159 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:57,742 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:58,163 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:59,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:59,469 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:59,743 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:24:00,165 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:01,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:02,171 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:03,174 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:04,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:05,144 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:05,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:06,176 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:07,167 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:08,176 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:09,171 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:10,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:24:10,136 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:24:10,177 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:10,211 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:11,171 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:12,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:13,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:14,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:15,174 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:16,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:16,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:17,182 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:18,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:19,182 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:20,177 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:21,183 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:22,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:22,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:23,186 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:24,181 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:24,797 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:24:25,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:24:25,133 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:24:25,185 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:26,179 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:27,184 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:27,208 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:28,189 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:29,188 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:30,185 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:31,190 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:32,189 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:33,152 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:33,187 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:34,191 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:35,190 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:36,191 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:37,190 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:38,191 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:39,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:39,200 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:40,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:24:40,132 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:24:40,201 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:41,203 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:42,204 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:43,205 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:44,203 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:44,227 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:45,211 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:46,206 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:47,208 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:48,214 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:49,209 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:50,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:50,210 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:51,210 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:52,211 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:53,210 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:54,214 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:54,798 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:24:55,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:24:55,133 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:24:55,192 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:55,213 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:56,214 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:57,210 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:58,213 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:59,220 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:00,223 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:01,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:01,219 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:02,216 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:03,216 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:04,218 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:05,220 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:06,226 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:07,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:07,223 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:08,228 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:09,225 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:10,142 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:25:10,142 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:25:10,237 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:11,233 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:11,766 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:25:12,227 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:12,264 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:13,237 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:14,236 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:15,256 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:16,257 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:17,264 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:18,160 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:18,246 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:19,253 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:20,253 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:21,248 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:22,252 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:23,047 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:25:23,047 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 with policy now
+2024-12-17 21:25:23,047 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:25:23,047 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:25:23,048 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:25:23,049 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:25:23,051 DEBUG   SenderThread:76461 [sender.py:send():379] send: exit
+2024-12-17 21:25:23,051 INFO    SenderThread:76461 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 21:25:23,051 INFO    SenderThread:76461 [sender.py:send_exit():588] handling runtime: 238
+2024-12-17 21:25:23,051 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:25:23,051 INFO    SenderThread:76461 [sender.py:send_exit():594] send defer
+2024-12-17 21:25:23,051 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:23,051 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 21:25:23,052 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:23,052 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 21:25:23,052 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 1
+2024-12-17 21:25:23,052 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:23,052 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 21:25:23,052 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:23,052 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 21:25:23,052 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 2
+2024-12-17 21:25:23,052 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:23,052 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 21:25:23,052 INFO    HandlerThread:76461 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 21:25:23,052 DEBUG   SystemMonitor:76461 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 21:25:23,052 INFO    HandlerThread:76461 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 21:25:23,052 DEBUG   SystemMonitor:76461 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 21:25:23,053 INFO    HandlerThread:76461 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 21:25:23,096 INFO    HandlerThread:76461 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 21:25:23,096 INFO    HandlerThread:76461 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 21:25:23,096 INFO    HandlerThread:76461 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 21:25:23,096 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:23,096 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 21:25:23,096 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 3
+2024-12-17 21:25:23,097 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:25:23,097 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:23,097 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 21:25:23,097 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:23,097 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 21:25:23,097 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 4
+2024-12-17 21:25:23,097 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:23,097 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 21:25:23,269 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/rrl4ps4z-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4
+2024-12-17 21:25:23,770 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:25:23,770 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:25:23,770 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4
+2024-12-17 21:25:23,771 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval
+2024-12-17 21:25:24,015 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/ipbifp7e-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4
+2024-12-17 21:25:28,586 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:28,586 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:28,586 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:28,587 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 21:25:28,587 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 5
+2024-12-17 21:25:28,587 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:28,587 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:28,587 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:25:28,587 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 21:25:28,587 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:25:28,588 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:25:28,588 DEBUG   SenderThread:76461 [sender.py:send():379] send: summary
+2024-12-17 21:25:28,588 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:25:28,588 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:28,588 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 21:25:28,588 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 6
+2024-12-17 21:25:28,588 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:28,588 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 21:25:28,589 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:28,589 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 21:25:28,589 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 7
+2024-12-17 21:25:28,589 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:28,589 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:28,589 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 21:25:28,589 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:28,589 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 21:25:28,772 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:25:29,052 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 21:25:29,052 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:29,669 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 8
+2024-12-17 21:25:29,669 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:29,669 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:29,669 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 21:25:29,670 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:29,670 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 21:25:29,670 INFO    SenderThread:76461 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 21:25:29,672 INFO    SenderThread:76461 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 21:25:29,686 INFO    SenderThread:76461 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 21:25:29,700 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 9
+2024-12-17 21:25:29,700 DEBUG   SenderThread:76461 [sender.py:send():379] send: artifact
+2024-12-17 21:25:29,700 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:29,702 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 21:25:29,772 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:25:30,053 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:30,189 INFO    wandb-upload_1:76461 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpat7ok6vh
+2024-12-17 21:25:30,320 INFO    wandb-upload_2:76461 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu793frvf
+2024-12-17 21:25:30,360 INFO    wandb-upload_0:76461 [upload_job.py:push():88] Uploaded file /tmp/tmp41i79nsx/wandb-job.json
+2024-12-17 21:25:30,913 INFO    SenderThread:76461 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzIyNjY3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzE5ODcyNA==', 'versionIndex': 378}}}
+2024-12-17 21:25:30,913 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:30,913 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 21:25:30,913 INFO    SenderThread:76461 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 21:25:31,773 INFO    SenderThread:76461 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files
+2024-12-17 21:25:31,773 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml config.yaml
+2024-12-17 21:25:31,773 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json wandb-summary.json
+2024-12-17 21:25:31,773 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:25:31,774 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt requirements.txt
+2024-12-17 21:25:31,774 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 21:25:31,774 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log output.log
+2024-12-17 21:25:31,774 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch diff.patch
+2024-12-17 21:25:31,777 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 21:25:31,777 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4
+2024-12-17 21:25:31,777 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4
+2024-12-17 21:25:31,778 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4
+2024-12-17 21:25:31,779 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4
+2024-12-17 21:25:31,780 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 10
+2024-12-17 21:25:31,780 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:31,782 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:31,782 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 21:25:31,782 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:31,782 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 21:25:31,782 INFO    SenderThread:76461 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:25:31,949 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:25:31,988 INFO    wandb-upload_1:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml
+2024-12-17 21:25:31,990 INFO    wandb-upload_3:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt
+2024-12-17 21:25:32,014 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/kg0vdiiy-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:25:32,036 INFO    wandb-upload_4:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:25:32,053 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:32,054 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:32,237 INFO    Thread-11 (_thread_body):76461 [sender.py:transition_state():614] send defer: 11
+2024-12-17 21:25:32,237 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:32,237 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 21:25:32,237 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:32,237 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 21:25:32,237 INFO    SenderThread:76461 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 21:25:32,238 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 12
+2024-12-17 21:25:32,238 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:32,238 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 21:25:32,238 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:32,238 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 21:25:32,238 INFO    SenderThread:76461 [file_stream.py:finish():601] file stream finish called
+2024-12-17 21:25:32,430 INFO    SenderThread:76461 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 21:25:32,430 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 13
+2024-12-17 21:25:32,430 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:32,430 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 21:25:32,430 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:32,430 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 21:25:32,430 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 14
+2024-12-17 21:25:32,430 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:32,431 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 21:25:32,431 DEBUG   SenderThread:76461 [sender.py:send():379] send: final
+2024-12-17 21:25:32,431 DEBUG   SenderThread:76461 [sender.py:send():379] send: footer
+2024-12-17 21:25:32,431 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:32,431 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 21:25:32,431 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:32,431 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 21:25:32,432 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:32,432 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:32,432 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: server_info
+2024-12-17 21:25:32,432 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 21:25:32,434 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 21:25:32,434 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:32,507 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:32,507 INFO    MainThread:76461 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 21:25:32,507 INFO    MainThread:76461 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 21:25:32,507 INFO    MainThread:76461 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 21:25:32,507 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 21:25:32,507 INFO    HandlerThread:76461 [handler.py:finish():882] shutting down handler
+2024-12-17 21:25:33,432 INFO    WriterThread:76461 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb
+2024-12-17 21:25:33,507 INFO    SenderThread:76461 [sender.py:finish():1615] shutting down sender
+2024-12-17 21:25:33,507 INFO    SenderThread:76461 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:25:33,507 INFO    SenderThread:76461 [file_pusher.py:join():175] waiting for file pusher

212118/wandb/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 21:21:24,425 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 21:21:24,425 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Configure stats pid to 76272
+2024-12-17 21:21:24,425 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/settings
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/logs/debug.log
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/logs/debug-internal.log
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:init():569] calling init triggers
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '523_0', 'wandb_notes': '523_0_req_603_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['fowlers_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118'}
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:init():619] starting backend
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:init():623] setting up manager
+2024-12-17 21:21:24,430 INFO    MainThread:76272 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 21:21:24,431 INFO    MainThread:76272 [wandb_init.py:init():631] backend started and connected
+2024-12-17 21:21:24,443 INFO    MainThread:76272 [wandb_init.py:init():720] updated telemetry
+2024-12-17 21:21:24,450 INFO    MainThread:76272 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 21:21:24,705 INFO    MainThread:76272 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 21:21:24,776 INFO    MainThread:76272 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 21:21:24,779 INFO    MainThread:76272 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 21:21:25,127 INFO    MainThread:76272 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 21:21:25,127 INFO    MainThread:76272 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 21:21:25,127 INFO    MainThread:76272 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 21:21:25,127 INFO    MainThread:76272 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 21:21:25,129 INFO    MainThread:76272 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 21:21:25,129 INFO    MainThread:76272 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/tb, True
+2024-12-17 21:21:31,696 INFO    MainThread:76272 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'}
+2024-12-17 21:25:33,510 WARNING MsgRouterThr:76272 [router.py:message_loop():77] message_loop has been closed

212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py

@@ -0,0 +1,512 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)		
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

212118/wandb/run-20241217_212124-78r9r705/files/config.yaml

@@ -0,0 +1,891 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '523_0'
+wandb_notes:
+  desc: null
+  value: 523_0_req_603_0
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value:
+  - fowlers_twodim
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 0
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/523_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1734488484
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: fowlers
+left_book_slot:
+  desc: null
+  value: twodim

212118/wandb/run-20241217_212124-78r9r705/files/diff.patch

@@ -0,0 +1,195 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index bb0a413..6b5d639 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -107,7 +107,7 @@ num_valid_demos: null
+ val_num_groups: 3
+ 
+ # name_of_expert_demo: 'book_insertion_wo_ends_wedge_v2_76_all_demos_10hz_no_filter_wo_flow_mask_expert_demos_imp_act'
+-name_of_expert_demo: '192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
++name_of_expert_demo: '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
+ expert_dataset_dirpath: '${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}'
+ 
+ expert_dataset: '${expert_dataset_dirpath}/demos.zarr'
+@@ -123,89 +123,28 @@ bc_weight_type: 'qfilter' # linear, qfilter
+ 
+ # Load weights
+ load_checkpoint: ${agent.load_checkpoint}
+-# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full 32 demos
+-# wandb_run_id: "265625" # RGBD history seed 0
+-# wandb_run_id: "266370_1" # RGBD history seed 1
+-# wandb_run_id: "266370_2" # RGBD history seed 2
+-# wandb_run_id: "266382_4" # RGBD history seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "266392_0" # RGBD+act history dataset seed 0
+-# wandb_run_id: "266407_0" # RGBD+act history dataset seed 1
+-wandb_run_id: "266406_0" # RGBD+act history dataset seed 2
+-
+-# wandb_run_id: "265626" # RGBD+Mask+act history seed 0
+-# wandb_run_id: "265638" # RGBD+Mask+act history seed 1
+-# wandb_run_id: "265637" # RGBD+Mask+act history seed 2
+-# wandb_run_id: "265636" # RGBD+Mask+act history seed 3
+-# wandb_run_id: "265635" # RGBD+Mask+act history seed 4
+-
+-# wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0
+-# wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1
+-# wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2
+-# wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3
+-# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0
+-# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1
+-# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2
+-# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0
+-# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1
+-# wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2
+-# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4
+-
+-
+-
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full dataset
+-# 6 step action history
+-# wandb_run_id: '250358_0' # RGBD+act history
+-# wandb_run_id: '250371_0' # RGBD+mask+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-# wandb_run_id: '250376_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 32 held out demos
+-# wandb_run_id: '250365_0' # RGBD+act history
+-# wandb_run_id: '250375_0' # RGBD+mask+act history
+-wandb_run_id: '250380_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250370_0' # RGBD+contact+act history
+-# 4 step action history
+-# wandb_run_id: '264772_0' # RGBD+act history
+-# wandb_run_id: '264775_0' # RGBD+mask+act history
+-# wandb_run_id: '264776_0' # RGBD+contact+act history
+-# wandb_run_id: '264777_0' # RGBD+mask+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
++# 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
++# 14 demos
++
++# RGBD+act history 
++# wandb_run_id: '497_0' # german
++# wandb_run_id: '505_0' # hbm
++# wandb_run_id: '506_0' # lady
++# wandb_run_id: '507_0' # greece
++# wandb_run_id: '508_0' # fowlers
++# wandb_run_id: '509_0' # lib
++# wandb_run_id: '510_0' # modelsys
++# wandb_run_id: '511_0' # electrodyn
++
++# RGBD+mask+act history
++# wandb_run_id: '512_0' # german
++# wandb_run_id: '520_0' # hbm
++# wandb_run_id: '521_0' # lady
++# wandb_run_id: '522_0' # greece
++wandb_run_id: '523_0' # fowlers
++# wandb_run_id: '524_0' # lib
++# wandb_run_id: '525_0' # modelsys
++# wandb_run_id: '526_0' # electrodyn
+ 
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 32 held out demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 9406a46..0f225ec 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -15,12 +15,24 @@ if not local_checkpoint_root_dir.exists():
+     local_checkpoint_root_dir.mkdir(parents=True)
+ 
+ # Download the model checkpoints
+-remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
+ remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser()
+ remote_username = 'leonmkim'
+-remote_host = 'halley'
++remote_host = 'grasp-login1'
+ 
+ #180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '512_0', # german
++#     '520_0', # hbm
++#     '521_0', # lady
++#     '522_0', # greece
++#     '523_0', # fowlers
++#     '524_0', # lib
++#     '525_0', # modelsys
++#     '526_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '265638',
+ #     '265637',
+@@ -75,6 +87,17 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '497_0', # german
++#     '505_0', # hbm
++#     '506_0', # lady
++#     '507_0', # greece
++#     '508_0', # fowlers
++#     '509_0', # lib
++#     '510_0', # modelsys
++#     '511_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '266382_4',
+ #     '266370_2',
+@@ -116,13 +139,13 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+-run_id_list = [
+-    # '266384_4,# not yet available
+-    '266372_1',
+-    '266372_2',
+-    '265627',
+-    '265620',
+-]
++# run_id_list = [
++#     # '266384_4,# not yet available
++#     '266372_1',
++#     '266372_2',
++#     '265627',
++#     '265620',
++# ]
+ checkpoint_epoch = 12000
+ 
+ # use subprocess to download the model checkpoints in parallel

212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7012418043e20ba472ca099eb2fd4bae9cf951ff5f200bf3b8b28bef7c455f17
+size 860246

212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a30824235f8d3481acdd75328ce66a4db21dec3ebd5b875ba7e3c527508d0f3
+size 854722

212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe502a2ffcc54aa7653539234f578d5a1a01cd0bf111c0cb8107c59433150318
+size 1236066

212118/wandb/run-20241217_212124-78r9r705/files/output.log

@@ -0,0 +1,3004 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1734488491.696665]: resetting environment
+[INFO] [1734488491.699516]: cleared current plan
+[INFO] [1734488491.700458]: moving to home
+[INFO] [1734488492.702628]: reached home
+[INFO] [1734488492.704727]: reset action history
+[INFO] [1734488495.414691]: environment reset
+Starting episode 0
+[INFO] [1734488495.416885]: resetting environment
+[INFO] [1734488495.420430]: cleared current plan
+[INFO] [1734488495.421459]: moving to home
+[INFO] [1734488496.424063]: reached home
+[INFO] [1734488496.425417]: reset action history
+[INFO] [1734488499.137813]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734488509.040145]: resetting environment
+[INFO] [1734488509.043179]: cleared current plan
+[INFO] [1734488509.044012]: moving to home
+[INFO] [1734488510.045332]: reached home
+[INFO] [1734488510.046555]: reset action history
+[INFO] [1734488512.753081]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/episode_rosbags/episode_0_2024-12-17-21-21-52.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-76272.log
+started roslaunch server http://158.130.50.37:42227/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [76587]
+process[print_text-2]: started with pid [76588]
+process[rosbag_record-3]: started with pid [76589]
+started bagging!
+[INFO] [1734488513.535930]: received mask_obs with header seq: 1330
+stamp:
+  secs: 1734488513
+  nsecs: 540523290
+frame_id: '' and grasped_object_mask header seq: 117429
+stamp:
+  secs: 1734488513
+  nsecs: 306998968
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488513.720297]: received mask_obs with header seq: 1331
+stamp:
+  secs: 1734488513
+  nsecs: 725090742
+frame_id: '' and grasped_object_mask header seq: 117435
+stamp:
+  secs: 1734488513
+  nsecs: 506836653
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488514.117301]: received mask_obs with header seq: 1332
+stamp:
+  secs: 1734488514
+  nsecs: 121912002
+frame_id: '' and grasped_object_mask header seq: 117447
+stamp:
+  secs: 1734488513
+  nsecs: 906508684
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488514.291310]: received mask_obs with header seq: 1333
+stamp:
+  secs: 1734488514
+  nsecs: 296135663
+frame_id: '' and grasped_object_mask header seq: 117452
+stamp:
+  secs: 1734488514
+  nsecs:  73037863
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488514.494223]: received mask_obs with header seq: 1334
+stamp:
+  secs: 1734488514
+  nsecs: 498311281
+frame_id: '' and grasped_object_mask header seq: 117457
+stamp:
+  secs: 1734488514
+  nsecs: 239567757
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488514.681076]: received mask_obs with header seq: 1335
+stamp:
+  secs: 1734488514
+  nsecs: 685832262
+frame_id: '' and grasped_object_mask header seq: 117463
+stamp:
+  secs: 1734488514
+  nsecs: 439402819
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488514.870616]: received mask_obs with header seq: 1336
+stamp:
+  secs: 1734488514
+  nsecs: 874481916
+frame_id: '' and grasped_object_mask header seq: 117469
+stamp:
+  secs: 1734488514
+  nsecs: 640603304
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488515.073492]: received mask_obs with header seq: 1337
+stamp:
+  secs: 1734488515
+  nsecs:  77445268
+frame_id: '' and grasped_object_mask header seq: 117475
+stamp:
+  secs: 1734488514
+  nsecs: 842766285
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488515.266401]: received mask_obs with header seq: 1338
+stamp:
+  secs: 1734488515
+  nsecs: 270193815
+frame_id: '' and grasped_object_mask header seq: 117480
+stamp:
+  secs: 1734488515
+  nsecs:  11301756
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488515.474418]: received mask_obs with header seq: 1339
+stamp:
+  secs: 1734488515
+  nsecs: 479188203
+frame_id: '' and grasped_object_mask header seq: 117486
+stamp:
+  secs: 1734488515
+  nsecs: 213622570
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488515.676012]: received mask_obs with header seq: 1340
+stamp:
+  secs: 1734488515
+  nsecs: 680783033
+frame_id: '' and grasped_object_mask header seq: 117493
+stamp:
+  secs: 1734488515
+  nsecs: 449772596
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488515.874816]: received mask_obs with header seq: 1341
+stamp:
+  secs: 1734488515
+  nsecs: 879640102
+frame_id: '' and grasped_object_mask header seq: 117499
+stamp:
+  secs: 1734488515
+  nsecs: 650627136
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488516.073933]: received mask_obs with header seq: 1342
+stamp:
+  secs: 1734488516
+  nsecs:  78693151
+frame_id: '' and grasped_object_mask header seq: 117505
+stamp:
+  secs: 1734488515
+  nsecs: 850232840
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488516.271909]: received mask_obs with header seq: 1343
+stamp:
+  secs: 1734488516
+  nsecs: 275898456
+frame_id: '' and grasped_object_mask header seq: 117511
+stamp:
+  secs: 1734488516
+  nsecs:  49826384
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488516.467535]: received mask_obs with header seq: 1344
+stamp:
+  secs: 1734488516
+  nsecs: 471306562
+frame_id: '' and grasped_object_mask header seq: 117516
+stamp:
+  secs: 1734488516
+  nsecs: 216145992
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488516.672146]: received mask_obs with header seq: 1345
+stamp:
+  secs: 1734488516
+  nsecs: 676180362
+frame_id: '' and grasped_object_mask header seq: 117522
+stamp:
+  secs: 1734488516
+  nsecs: 415718079
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488516.875914]: received mask_obs with header seq: 1346
+stamp:
+  secs: 1734488516
+  nsecs: 880681514
+frame_id: '' and grasped_object_mask header seq: 117528
+stamp:
+  secs: 1734488516
+  nsecs: 615297556
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488517.081978]: received mask_obs with header seq: 1347
+stamp:
+  secs: 1734488517
+  nsecs:  86817026
+frame_id: '' and grasped_object_mask header seq: 117535
+stamp:
+  secs: 1734488516
+  nsecs: 848172426
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488517.279205]: received mask_obs with header seq: 1348
+stamp:
+  secs: 1734488517
+  nsecs: 283457756
+frame_id: '' and grasped_object_mask header seq: 117541
+stamp:
+  secs: 1734488517
+  nsecs:  47765970
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488517.476432]: received mask_obs with header seq: 1349
+stamp:
+  secs: 1734488517
+  nsecs: 481223821
+frame_id: '' and grasped_object_mask header seq: 117547
+stamp:
+  secs: 1734488517
+  nsecs: 247347593
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488517.673335]: received mask_obs with header seq: 1350
+stamp:
+  secs: 1734488517
+  nsecs: 678065299
+frame_id: '' and grasped_object_mask header seq: 117553
+stamp:
+  secs: 1734488517
+  nsecs: 446918726
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488517.871731]: received mask_obs with header seq: 1351
+stamp:
+  secs: 1734488517
+  nsecs: 876430988
+frame_id: '' and grasped_object_mask header seq: 117559
+stamp:
+  secs: 1734488517
+  nsecs: 646504879
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488518.076064]: received mask_obs with header seq: 1352
+stamp:
+  secs: 1734488518
+  nsecs:  80809831
+frame_id: '' and grasped_object_mask header seq: 117564
+stamp:
+  secs: 1734488517
+  nsecs: 812840939
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488518.276494]: received mask_obs with header seq: 1353
+stamp:
+  secs: 1734488518
+  nsecs: 281139850
+frame_id: '' and grasped_object_mask header seq: 117571
+stamp:
+  secs: 1734488518
+  nsecs:  45698166
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488518.475418]: received mask_obs with header seq: 1354
+stamp:
+  secs: 1734488518
+  nsecs: 480235338
+frame_id: '' and grasped_object_mask header seq: 117577
+stamp:
+  secs: 1734488518
+  nsecs: 245277166
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488518.670695]: received mask_obs with header seq: 1355
+stamp:
+  secs: 1734488518
+  nsecs: 675471067
+frame_id: '' and grasped_object_mask header seq: 117583
+stamp:
+  secs: 1734488518
+  nsecs: 444844007
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488518.858743]: received mask_obs with header seq: 1356
+stamp:
+  secs: 1734488518
+  nsecs: 863535881
+frame_id: '' and grasped_object_mask header seq: 117588
+stamp:
+  secs: 1734488518
+  nsecs: 611164093
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488519.050615]: received mask_obs with header seq: 1357
+stamp:
+  secs: 1734488519
+  nsecs:  55445671
+frame_id: '' and grasped_object_mask header seq: 117594
+stamp:
+  secs: 1734488518
+  nsecs: 810777903
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488519.245232]: received mask_obs with header seq: 1358
+stamp:
+  secs: 1734488519
+  nsecs: 249953031
+frame_id: '' and grasped_object_mask header seq: 117600
+stamp:
+  secs: 1734488519
+  nsecs:  10380030
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488519.442395]: received mask_obs with header seq: 1359
+stamp:
+  secs: 1734488519
+  nsecs: 447017192
+frame_id: '' and grasped_object_mask header seq: 117606
+stamp:
+  secs: 1734488519
+  nsecs: 209971666
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488519.640498]: received mask_obs with header seq: 1360
+stamp:
+  secs: 1734488519
+  nsecs: 645313978
+frame_id: '' and grasped_object_mask header seq: 117612
+stamp:
+  secs: 1734488519
+  nsecs: 409551144
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488519.837375]: received mask_obs with header seq: 1361
+stamp:
+  secs: 1734488519
+  nsecs: 842097043
+frame_id: '' and grasped_object_mask header seq: 117618
+stamp:
+  secs: 1734488519
+  nsecs: 609138727
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488520.037435]: received mask_obs with header seq: 1362
+stamp:
+  secs: 1734488520
+  nsecs:  41568517
+frame_id: '' and grasped_object_mask header seq: 117624
+stamp:
+  secs: 1734488519
+  nsecs: 808754683
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488520.230878]: received mask_obs with header seq: 1363
+stamp:
+  secs: 1734488520
+  nsecs: 235653162
+frame_id: '' and grasped_object_mask header seq: 117629
+stamp:
+  secs: 1734488519
+  nsecs: 975093365
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488520.417156]: received mask_obs with header seq: 1364
+stamp:
+  secs: 1734488520
+  nsecs: 421958446
+frame_id: '' and grasped_object_mask header seq: 117635
+stamp:
+  secs: 1734488520
+  nsecs: 174688816
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488520.613280]: received mask_obs with header seq: 1365
+stamp:
+  secs: 1734488520
+  nsecs: 617956876
+frame_id: '' and grasped_object_mask header seq: 117641
+stamp:
+  secs: 1734488520
+  nsecs: 374273062
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488520.806447]: received mask_obs with header seq: 1366
+stamp:
+  secs: 1734488520
+  nsecs: 811276197
+frame_id: '' and grasped_object_mask header seq: 117647
+stamp:
+  secs: 1734488520
+  nsecs: 573854208
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488521.005897]: received mask_obs with header seq: 1367
+stamp:
+  secs: 1734488521
+  nsecs:  10637044
+frame_id: '' and grasped_object_mask header seq: 117653
+stamp:
+  secs: 1734488520
+  nsecs: 773466349
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488521.203162]: received mask_obs with header seq: 1368
+stamp:
+  secs: 1734488521
+  nsecs: 207555055
+frame_id: '' and grasped_object_mask header seq: 117659
+stamp:
+  secs: 1734488520
+  nsecs: 973066807
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488521.393816]: received mask_obs with header seq: 1369
+stamp:
+  secs: 1734488521
+  nsecs: 398211240
+frame_id: '' and grasped_object_mask header seq: 117664
+stamp:
+  secs: 1734488521
+  nsecs: 139392614
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488521.583850]: received mask_obs with header seq: 1370
+stamp:
+  secs: 1734488521
+  nsecs: 588620901
+frame_id: '' and grasped_object_mask header seq: 117670
+stamp:
+  secs: 1734488521
+  nsecs: 338971853
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488521.782108]: received mask_obs with header seq: 1371
+stamp:
+  secs: 1734488521
+  nsecs: 786865711
+frame_id: '' and grasped_object_mask header seq: 117676
+stamp:
+  secs: 1734488521
+  nsecs: 538539886
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488521.975370]: received mask_obs with header seq: 1372
+stamp:
+  secs: 1734488521
+  nsecs: 980169534
+frame_id: '' and grasped_object_mask header seq: 117682
+stamp:
+  secs: 1734488521
+  nsecs: 738153458
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488522.173034]: received mask_obs with header seq: 1373
+stamp:
+  secs: 1734488522
+  nsecs: 177629947
+frame_id: '' and grasped_object_mask header seq: 117688
+stamp:
+  secs: 1734488521
+  nsecs: 937754631
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488522.373051]: received mask_obs with header seq: 1374
+stamp:
+  secs: 1734488522
+  nsecs: 377842187
+frame_id: '' and grasped_object_mask header seq: 117694
+stamp:
+  secs: 1734488522
+  nsecs: 137343645
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488522.562010]: received mask_obs with header seq: 1375
+stamp:
+  secs: 1734488522
+  nsecs: 566809654
+frame_id: '' and grasped_object_mask header seq: 117699
+stamp:
+  secs: 1734488522
+  nsecs: 303659678
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488522.760386]: received mask_obs with header seq: 1376
+stamp:
+  secs: 1734488522
+  nsecs: 765190124
+frame_id: '' and grasped_object_mask header seq: 117706
+stamp:
+  secs: 1734488522
+  nsecs: 536486864
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488522.959358]: received mask_obs with header seq: 1377
+stamp:
+  secs: 1734488522
+  nsecs: 963238239
+frame_id: '' and grasped_object_mask header seq: 117712
+stamp:
+  secs: 1734488522
+  nsecs: 736104250
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488523.166117]: received mask_obs with header seq: 1378
+stamp:
+  secs: 1734488523
+  nsecs: 170861482
+frame_id: '' and grasped_object_mask header seq: 117718
+stamp:
+  secs: 1734488522
+  nsecs: 935710907
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488523.363388]: received mask_obs with header seq: 1379
+stamp:
+  secs: 1734488523
+  nsecs: 368171215
+frame_id: '' and grasped_object_mask header seq: 117724
+stamp:
+  secs: 1734488523
+  nsecs: 135305405
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488523.563067]: received mask_obs with header seq: 1380
+stamp:
+  secs: 1734488523
+  nsecs: 567840337
+frame_id: '' and grasped_object_mask header seq: 117730
+stamp:
+  secs: 1734488523
+  nsecs: 334888458
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488523.760427]: received mask_obs with header seq: 1381
+stamp:
+  secs: 1734488523
+  nsecs: 764970064
+frame_id: '' and grasped_object_mask header seq: 117736
+stamp:
+  secs: 1734488523
+  nsecs: 534459829
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488523.957353]: received mask_obs with header seq: 1382
+stamp:
+  secs: 1734488523
+  nsecs: 962140083
+frame_id: '' and grasped_object_mask header seq: 117742
+stamp:
+  secs: 1734488523
+  nsecs: 734083652
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488524.152965]: received mask_obs with header seq: 1383
+stamp:
+  secs: 1734488524
+  nsecs: 157132863
+frame_id: '' and grasped_object_mask header seq: 117747
+stamp:
+  secs: 1734488523
+  nsecs: 900429010
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488524.342782]: received mask_obs with header seq: 1384
+stamp:
+  secs: 1734488524
+  nsecs: 347471475
+frame_id: '' and grasped_object_mask header seq: 117753
+stamp:
+  secs: 1734488524
+  nsecs: 100032806
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488524.537917]: received mask_obs with header seq: 1385
+stamp:
+  secs: 1734488524
+  nsecs: 542565584
+frame_id: '' and grasped_object_mask header seq: 117759
+stamp:
+  secs: 1734488524
+  nsecs: 299625397
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488524.737870]: received mask_obs with header seq: 1386
+stamp:
+  secs: 1734488524
+  nsecs: 742503643
+frame_id: '' and grasped_object_mask header seq: 117765
+stamp:
+  secs: 1734488524
+  nsecs: 499207258
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488524.938627]: received mask_obs with header seq: 1387
+stamp:
+  secs: 1734488524
+  nsecs: 943261861
+frame_id: '' and grasped_object_mask header seq: 117771
+stamp:
+  secs: 1734488524
+  nsecs: 698829889
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488525.151322]: received mask_obs with header seq: 1388
+stamp:
+  secs: 1734488525
+  nsecs: 156005382
+frame_id: '' and grasped_object_mask header seq: 117777
+stamp:
+  secs: 1734488524
+  nsecs: 898440123
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488525.343674]: received mask_obs with header seq: 1389
+stamp:
+  secs: 1734488525
+  nsecs: 348135948
+frame_id: '' and grasped_object_mask header seq: 117783
+stamp:
+  secs: 1734488525
+  nsecs:  98038435
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488525.540274]: received mask_obs with header seq: 1390
+stamp:
+  secs: 1734488525
+  nsecs: 545060396
+frame_id: '' and grasped_object_mask header seq: 117789
+stamp:
+  secs: 1734488525
+  nsecs: 297625065
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488525.737348]: received mask_obs with header seq: 1391
+stamp:
+  secs: 1734488525
+  nsecs: 741950035
+frame_id: '' and grasped_object_mask header seq: 117795
+stamp:
+  secs: 1734488525
+  nsecs: 497200489
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488525.950629]: received mask_obs with header seq: 1392
+stamp:
+  secs: 1734488525
+  nsecs: 955370903
+frame_id: '' and grasped_object_mask header seq: 117801
+stamp:
+  secs: 1734488525
+  nsecs: 696797848
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488526.139287]: received mask_obs with header seq: 1393
+stamp:
+  secs: 1734488526
+  nsecs: 143530845
+frame_id: '' and grasped_object_mask header seq: 117807
+stamp:
+  secs: 1734488525
+  nsecs: 896395445
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488526.354287]: received mask_obs with header seq: 1394
+stamp:
+  secs: 1734488526
+  nsecs: 359120130
+frame_id: '' and grasped_object_mask header seq: 117813
+stamp:
+  secs: 1734488526
+  nsecs:  95981836
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488526.544801]: received mask_obs with header seq: 1395
+stamp:
+  secs: 1734488526
+  nsecs: 549009561
+frame_id: '' and grasped_object_mask header seq: 117819
+stamp:
+  secs: 1734488526
+  nsecs: 295555353
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488526.737395]: received mask_obs with header seq: 1396
+stamp:
+  secs: 1734488526
+  nsecs: 742144584
+frame_id: '' and grasped_object_mask header seq: 117825
+stamp:
+  secs: 1734488526
+  nsecs: 495116949
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488526.937526]: received mask_obs with header seq: 1397
+stamp:
+  secs: 1734488526
+  nsecs: 941965341
+frame_id: '' and grasped_object_mask header seq: 117831
+stamp:
+  secs: 1734488526
+  nsecs: 694713354
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488527.135987]: received mask_obs with header seq: 1398
+stamp:
+  secs: 1734488527
+  nsecs: 140719413
+frame_id: '' and grasped_object_mask header seq: 117837
+stamp:
+  secs: 1734488526
+  nsecs: 894314528
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488527.337024]: received mask_obs with header seq: 1399
+stamp:
+  secs: 1734488527
+  nsecs: 341829061
+frame_id: '' and grasped_object_mask header seq: 117843
+stamp:
+  secs: 1734488527
+  nsecs:  93903780
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488527.536707]: received mask_obs with header seq: 1400
+stamp:
+  secs: 1734488527
+  nsecs: 541493177
+frame_id: '' and grasped_object_mask header seq: 117849
+stamp:
+  secs: 1734488527
+  nsecs: 293482304
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488527.735802]: received mask_obs with header seq: 1401
+stamp:
+  secs: 1734488527
+  nsecs: 740492105
+frame_id: '' and grasped_object_mask header seq: 117855
+stamp:
+  secs: 1734488527
+  nsecs: 493048191
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488527.936937]: received mask_obs with header seq: 1402
+stamp:
+  secs: 1734488527
+  nsecs: 941634416
+frame_id: '' and grasped_object_mask header seq: 117861
+stamp:
+  secs: 1734488527
+  nsecs: 692643404
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488528.136992]: received mask_obs with header seq: 1403
+stamp:
+  secs: 1734488528
+  nsecs: 141670703
+frame_id: '' and grasped_object_mask header seq: 117867
+stamp:
+  secs: 1734488527
+  nsecs: 892241240
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488528.336021]: received mask_obs with header seq: 1404
+stamp:
+  secs: 1734488528
+  nsecs: 340800285
+frame_id: '' and grasped_object_mask header seq: 117873
+stamp:
+  secs: 1734488528
+  nsecs:  91827154
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488528.536392]: received mask_obs with header seq: 1405
+stamp:
+  secs: 1734488528
+  nsecs: 541168689
+frame_id: '' and grasped_object_mask header seq: 117879
+stamp:
+  secs: 1734488528
+  nsecs: 291401148
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488528.740782]: received mask_obs with header seq: 1406
+stamp:
+  secs: 1734488528
+  nsecs: 745445489
+frame_id: '' and grasped_object_mask header seq: 117885
+stamp:
+  secs: 1734488528
+  nsecs: 490964174
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488528.937434]: received mask_obs with header seq: 1407
+stamp:
+  secs: 1734488528
+  nsecs: 942167997
+frame_id: '' and grasped_object_mask header seq: 117891
+stamp:
+  secs: 1734488528
+  nsecs: 690561056
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488529.155751]: received mask_obs with header seq: 1408
+stamp:
+  secs: 1734488529
+  nsecs: 160276651
+frame_id: '' and grasped_object_mask header seq: 117897
+stamp:
+  secs: 1734488528
+  nsecs: 890163422
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488529.355766]: received mask_obs with header seq: 1409
+stamp:
+  secs: 1734488529
+  nsecs: 360211610
+frame_id: '' and grasped_object_mask header seq: 117904
+stamp:
+  secs: 1734488529
+  nsecs: 123018265
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488529.553316]: received mask_obs with header seq: 1410
+stamp:
+  secs: 1734488529
+  nsecs: 557853460
+frame_id: '' and grasped_object_mask header seq: 117910
+stamp:
+  secs: 1734488529
+  nsecs: 322594881
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488529.749020]: received mask_obs with header seq: 1411
+stamp:
+  secs: 1734488529
+  nsecs: 753644466
+frame_id: '' and grasped_object_mask header seq: 117915
+stamp:
+  secs: 1734488529
+  nsecs: 488899469
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488529.938665]: received mask_obs with header seq: 1412
+stamp:
+  secs: 1734488529
+  nsecs: 943445920
+frame_id: '' and grasped_object_mask header seq: 117921
+stamp:
+  secs: 1734488529
+  nsecs: 688495874
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488530.138073]: received mask_obs with header seq: 1413
+stamp:
+  secs: 1734488530
+  nsecs: 142867565
+frame_id: '' and grasped_object_mask header seq: 117927
+stamp:
+  secs: 1734488529
+  nsecs: 888093948
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488530.337759]: received mask_obs with header seq: 1414
+stamp:
+  secs: 1734488530
+  nsecs: 342407464
+frame_id: '' and grasped_object_mask header seq: 117933
+stamp:
+  secs: 1734488530
+  nsecs:  87680101
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488530.537737]: received mask_obs with header seq: 1415
+stamp:
+  secs: 1734488530
+  nsecs: 542540550
+frame_id: '' and grasped_object_mask header seq: 117939
+stamp:
+  secs: 1734488530
+  nsecs: 287254333
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488530.737130]: received mask_obs with header seq: 1416
+stamp:
+  secs: 1734488530
+  nsecs: 741718053
+frame_id: '' and grasped_object_mask header seq: 117945
+stamp:
+  secs: 1734488530
+  nsecs: 486816645
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488530.937474]: received mask_obs with header seq: 1417
+stamp:
+  secs: 1734488530
+  nsecs: 942238807
+frame_id: '' and grasped_object_mask header seq: 117951
+stamp:
+  secs: 1734488530
+  nsecs: 687331676
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488531.137755]: received mask_obs with header seq: 1418
+stamp:
+  secs: 1734488531
+  nsecs: 142568349
+frame_id: '' and grasped_object_mask header seq: 117957
+stamp:
+  secs: 1734488530
+  nsecs: 888279915
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488531.341727]: received mask_obs with header seq: 1419
+stamp:
+  secs: 1734488531
+  nsecs: 344391107
+frame_id: '' and grasped_object_mask header seq: 117963
+stamp:
+  secs: 1734488531
+  nsecs:  89315891
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488531.537701]: received mask_obs with header seq: 1420
+stamp:
+  secs: 1734488531
+  nsecs: 542112588
+frame_id: '' and grasped_object_mask header seq: 117969
+stamp:
+  secs: 1734488531
+  nsecs: 290436983
+frame_id: "camera_color_optical_frame"
+preempting episode
+[INFO] [1734488531.601489]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1734488531.755733]: received mask_obs with header seq: 1421
+stamp:
+  secs: 1734488531
+  nsecs: 760311365
+frame_id: '' and grasped_object_mask header seq: 117975
+stamp:
+  secs: 1734488531
+  nsecs: 491644859
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488532.394552]: Stored episode 1.
+[INFO] [1734488532.394810]: Saving video...
+[INFO] [1734488532.586377]: Video saved!
+Starting episode 1
+[INFO] [1734488532.600132]: resetting environment
+[INFO] [1734488532.609962]: cleared current plan
+[INFO] [1734488532.611135]: moving to home
+[INFO] [1734488535.913657]: reached home
+[INFO] [1734488535.914433]: reset action history
+[INFO] [1734488538.624623]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734488557.040617]: resetting environment
+[INFO] [1734488557.043843]: cleared current plan
+[INFO] [1734488557.044026]: moving to home
+[INFO] [1734488558.045287]: reached home
+[INFO] [1734488558.046190]: reset action history
+[INFO] [1734488560.753731]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/episode_rosbags/episode_1_2024-12-17-21-22-40.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-76272.log
+started roslaunch server http://158.130.50.37:44067/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [76713]
+process[print_text-5]: started with pid [76714]
+process[rosbag_record-6]: started with pid [76715]
+started bagging!
+[INFO] [1734488561.427042]: received mask_obs with header seq: 1422
+stamp:
+  secs: 1734488561
+  nsecs: 431577920
+frame_id: '' and grasped_object_mask header seq: 118867
+stamp:
+  secs: 1734488561
+  nsecs: 201576233
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488561.622701]: received mask_obs with header seq: 1423
+stamp:
+  secs: 1734488561
+  nsecs: 627487421
+frame_id: '' and grasped_object_mask header seq: 118873
+stamp:
+  secs: 1734488561
+  nsecs: 401416063
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488561.821698]: received mask_obs with header seq: 1424
+stamp:
+  secs: 1734488561
+  nsecs: 826492071
+frame_id: '' and grasped_object_mask header seq: 118879
+stamp:
+  secs: 1734488561
+  nsecs: 601254463
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488562.032444]: received mask_obs with header seq: 1425
+stamp:
+  secs: 1734488562
+  nsecs:  37048578
+frame_id: '' and grasped_object_mask header seq: 118885
+stamp:
+  secs: 1734488561
+  nsecs: 801090956
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is 118845038 for nearest: 1734488562119771771 - target: 1734488562000926733 at idx 15
+[INFO] [1734488562.249000]: received mask_obs with header seq: 1426
+stamp:
+  secs: 1734488562
+  nsecs: 252330064
+frame_id: '' and grasped_object_mask header seq: 118891
+stamp:
+  secs: 1734488562
+  nsecs:    926733
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488562.452958]: received mask_obs with header seq: 1427
+stamp:
+  secs: 1734488562
+  nsecs: 457753896
+frame_id: '' and grasped_object_mask header seq: 118898
+stamp:
+  secs: 1734488562
+  nsecs: 234067678
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488562.653386]: received mask_obs with header seq: 1428
+stamp:
+  secs: 1734488562
+  nsecs: 657989740
+frame_id: '' and grasped_object_mask header seq: 118904
+stamp:
+  secs: 1734488562
+  nsecs: 433904171
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488562.852416]: received mask_obs with header seq: 1429
+stamp:
+  secs: 1734488562
+  nsecs: 856615066
+frame_id: '' and grasped_object_mask header seq: 118909
+stamp:
+  secs: 1734488562
+  nsecs: 600434542
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488563.044461]: received mask_obs with header seq: 1430
+stamp:
+  secs: 1734488563
+  nsecs:  49080848
+frame_id: '' and grasped_object_mask header seq: 118915
+stamp:
+  secs: 1734488562
+  nsecs: 800274849
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488563.244241]: received mask_obs with header seq: 1431
+stamp:
+  secs: 1734488563
+  nsecs: 249041557
+frame_id: '' and grasped_object_mask header seq: 118921
+stamp:
+  secs: 1734488563
+  nsecs:    116348
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488563.443369]: received mask_obs with header seq: 1432
+stamp:
+  secs: 1734488563
+  nsecs: 447961330
+frame_id: '' and grasped_object_mask header seq: 118927
+stamp:
+  secs: 1734488563
+  nsecs: 199957132
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488563.643313]: received mask_obs with header seq: 1433
+stamp:
+  secs: 1734488563
+  nsecs: 648080587
+frame_id: '' and grasped_object_mask header seq: 118933
+stamp:
+  secs: 1734488563
+  nsecs: 399798155
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488563.842666]: received mask_obs with header seq: 1434
+stamp:
+  secs: 1734488563
+  nsecs: 847209453
+frame_id: '' and grasped_object_mask header seq: 118939
+stamp:
+  secs: 1734488563
+  nsecs: 599638939
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488564.063322]: received mask_obs with header seq: 1435
+stamp:
+  secs: 1734488564
+  nsecs:  67942380
+frame_id: '' and grasped_object_mask header seq: 118945
+stamp:
+  secs: 1734488563
+  nsecs: 799479485
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488564.266694]: received mask_obs with header seq: 1436
+stamp:
+  secs: 1734488564
+  nsecs: 271458625
+frame_id: '' and grasped_object_mask header seq: 118952
+stamp:
+  secs: 1734488564
+  nsecs:  32627344
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488564.464137]: received mask_obs with header seq: 1437
+stamp:
+  secs: 1734488564
+  nsecs: 468806505
+frame_id: '' and grasped_object_mask header seq: 118958
+stamp:
+  secs: 1734488564
+  nsecs: 232468367
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488564.662186]: received mask_obs with header seq: 1438
+stamp:
+  secs: 1734488564
+  nsecs: 666749000
+frame_id: '' and grasped_object_mask header seq: 118964
+stamp:
+  secs: 1734488564
+  nsecs: 432309151
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488564.856328]: received mask_obs with header seq: 1439
+stamp:
+  secs: 1734488564
+  nsecs: 861112833
+frame_id: '' and grasped_object_mask header seq: 118970
+stamp:
+  secs: 1734488564
+  nsecs: 632148027
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488565.057206]: received mask_obs with header seq: 1440
+stamp:
+  secs: 1734488565
+  nsecs:  61779737
+frame_id: '' and grasped_object_mask header seq: 118976
+stamp:
+  secs: 1734488564
+  nsecs: 831984520
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488565.251659]: received mask_obs with header seq: 1441
+stamp:
+  secs: 1734488565
+  nsecs: 256386041
+frame_id: '' and grasped_object_mask header seq: 118981
+stamp:
+  secs: 1734488564
+  nsecs: 998513699
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488565.446504]: received mask_obs with header seq: 1442
+stamp:
+  secs: 1734488565
+  nsecs: 451287508
+frame_id: '' and grasped_object_mask header seq: 118987
+stamp:
+  secs: 1734488565
+  nsecs: 198348761
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488565.643145]: received mask_obs with header seq: 1443
+stamp:
+  secs: 1734488565
+  nsecs: 647970676
+frame_id: '' and grasped_object_mask header seq: 118993
+stamp:
+  secs: 1734488565
+  nsecs: 398184061
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488565.846698]: received mask_obs with header seq: 1444
+stamp:
+  secs: 1734488565
+  nsecs: 851200580
+frame_id: '' and grasped_object_mask header seq: 118999
+stamp:
+  secs: 1734488565
+  nsecs: 598020077
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488566.047535]: received mask_obs with header seq: 1445
+stamp:
+  secs: 1734488566
+  nsecs:  51820993
+frame_id: '' and grasped_object_mask header seq: 119005
+stamp:
+  secs: 1734488565
+  nsecs: 797855377
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488566.243914]: received mask_obs with header seq: 1446
+stamp:
+  secs: 1734488566
+  nsecs: 248727083
+frame_id: '' and grasped_object_mask header seq: 119011
+stamp:
+  secs: 1734488565
+  nsecs: 997690916
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488566.444379]: received mask_obs with header seq: 1447
+stamp:
+  secs: 1734488566
+  nsecs: 449194192
+frame_id: '' and grasped_object_mask header seq: 119017
+stamp:
+  secs: 1734488566
+  nsecs: 197526217
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488566.649732]: received mask_obs with header seq: 1448
+stamp:
+  secs: 1734488566
+  nsecs: 654295206
+frame_id: '' and grasped_object_mask header seq: 119023
+stamp:
+  secs: 1734488566
+  nsecs: 397361279
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488566.848263]: received mask_obs with header seq: 1449
+stamp:
+  secs: 1734488566
+  nsecs: 852911710
+frame_id: '' and grasped_object_mask header seq: 119029
+stamp:
+  secs: 1734488566
+  nsecs: 597196817
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488567.047250]: received mask_obs with header seq: 1450
+stamp:
+  secs: 1734488567
+  nsecs:  51189422
+frame_id: '' and grasped_object_mask header seq: 119035
+stamp:
+  secs: 1734488566
+  nsecs: 797034979
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488567.246892]: received mask_obs with header seq: 1451
+stamp:
+  secs: 1734488567
+  nsecs: 251092672
+frame_id: '' and grasped_object_mask header seq: 119041
+stamp:
+  secs: 1734488566
+  nsecs: 996872187
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488567.453555]: received mask_obs with header seq: 1452
+stamp:
+  secs: 1734488567
+  nsecs: 456952571
+frame_id: '' and grasped_object_mask header seq: 119047
+stamp:
+  secs: 1734488567
+  nsecs: 196709156
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488567.656929]: received mask_obs with header seq: 1453
+stamp:
+  secs: 1734488567
+  nsecs: 660795450
+frame_id: '' and grasped_object_mask header seq: 119053
+stamp:
+  secs: 1734488567
+  nsecs: 396546364
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488567.856830]: received mask_obs with header seq: 1454
+stamp:
+  secs: 1734488567
+  nsecs: 860838890
+frame_id: '' and grasped_object_mask header seq: 119060
+stamp:
+  secs: 1734488567
+  nsecs: 629689455
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488568.060009]: received mask_obs with header seq: 1455
+stamp:
+  secs: 1734488568
+  nsecs:  64021348
+frame_id: '' and grasped_object_mask header seq: 119065
+stamp:
+  secs: 1734488567
+  nsecs: 796217680
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488568.260677]: received mask_obs with header seq: 1456
+stamp:
+  secs: 1734488568
+  nsecs: 262897968
+frame_id: '' and grasped_object_mask header seq: 119072
+stamp:
+  secs: 1734488568
+  nsecs:  29358387
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488568.457087]: received mask_obs with header seq: 1457
+stamp:
+  secs: 1734488568
+  nsecs: 461876392
+frame_id: '' and grasped_object_mask header seq: 119078
+stamp:
+  secs: 1734488568
+  nsecs: 229192734
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488568.653156]: received mask_obs with header seq: 1458
+stamp:
+  secs: 1734488568
+  nsecs: 657963752
+frame_id: '' and grasped_object_mask header seq: 119083
+stamp:
+  secs: 1734488568
+  nsecs: 395721674
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488568.846745]: received mask_obs with header seq: 1459
+stamp:
+  secs: 1734488568
+  nsecs: 850787162
+frame_id: '' and grasped_object_mask header seq: 119089
+stamp:
+  secs: 1734488568
+  nsecs: 595556736
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488569.046436]: received mask_obs with header seq: 1460
+stamp:
+  secs: 1734488569
+  nsecs:  50691604
+frame_id: '' and grasped_object_mask header seq: 119095
+stamp:
+  secs: 1734488568
+  nsecs: 795397043
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488569.243300]: received mask_obs with header seq: 1461
+stamp:
+  secs: 1734488569
+  nsecs: 248096227
+frame_id: '' and grasped_object_mask header seq: 119101
+stamp:
+  secs: 1734488568
+  nsecs: 995237589
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488569.452471]: received mask_obs with header seq: 1462
+stamp:
+  secs: 1734488569
+  nsecs: 457210063
+frame_id: '' and grasped_object_mask header seq: 119107
+stamp:
+  secs: 1734488569
+  nsecs: 195079088
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488569.660160]: received mask_obs with header seq: 1463
+stamp:
+  secs: 1734488569
+  nsecs: 664606571
+frame_id: '' and grasped_object_mask header seq: 119113
+stamp:
+  secs: 1734488569
+  nsecs: 394919634
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488569.859895]: received mask_obs with header seq: 1464
+stamp:
+  secs: 1734488569
+  nsecs: 863925695
+frame_id: '' and grasped_object_mask header seq: 119120
+stamp:
+  secs: 1734488569
+  nsecs: 628063917
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488570.058305]: received mask_obs with header seq: 1465
+stamp:
+  secs: 1734488570
+  nsecs:  63014030
+frame_id: '' and grasped_object_mask header seq: 119126
+stamp:
+  secs: 1734488569
+  nsecs: 827895403
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488570.255535]: received mask_obs with header seq: 1466
+stamp:
+  secs: 1734488570
+  nsecs: 260270357
+frame_id: '' and grasped_object_mask header seq: 119132
+stamp:
+  secs: 1734488570
+  nsecs:  27725935
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488570.448285]: received mask_obs with header seq: 1467
+stamp:
+  secs: 1734488570
+  nsecs: 453035354
+frame_id: '' and grasped_object_mask header seq: 119137
+stamp:
+  secs: 1734488570
+  nsecs: 194250584
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488570.649887]: received mask_obs with header seq: 1468
+stamp:
+  secs: 1734488570
+  nsecs: 654668092
+frame_id: '' and grasped_object_mask header seq: 119143
+stamp:
+  secs: 1734488570
+  nsecs: 394081116
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488570.851192]: received mask_obs with header seq: 1469
+stamp:
+  secs: 1734488570
+  nsecs: 855838060
+frame_id: '' and grasped_object_mask header seq: 119149
+stamp:
+  secs: 1734488570
+  nsecs: 593911171
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488571.051944]: received mask_obs with header seq: 1470
+stamp:
+  secs: 1734488571
+  nsecs:  56265830
+frame_id: '' and grasped_object_mask header seq: 119155
+stamp:
+  secs: 1734488570
+  nsecs: 793743849
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488571.250141]: received mask_obs with header seq: 1471
+stamp:
+  secs: 1734488571
+  nsecs: 254881858
+frame_id: '' and grasped_object_mask header seq: 119161
+stamp:
+  secs: 1734488570
+  nsecs: 993577003
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488571.449057]: received mask_obs with header seq: 1472
+stamp:
+  secs: 1734488571
+  nsecs: 453782796
+frame_id: '' and grasped_object_mask header seq: 119167
+stamp:
+  secs: 1734488571
+  nsecs: 193409443
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488571.649302]: received mask_obs with header seq: 1473
+stamp:
+  secs: 1734488571
+  nsecs: 654060363
+frame_id: '' and grasped_object_mask header seq: 119174
+stamp:
+  secs: 1734488571
+  nsecs: 426547527
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -106604662 for nearest: 1734488571519776974 - target: 1734488571626381636 at idx 146
+[INFO] [1734488571.846306]: received mask_obs with header seq: 1474
+stamp:
+  secs: 1734488571
+  nsecs: 849535703
+frame_id: '' and grasped_object_mask header seq: 119179
+stamp:
+  secs: 1734488571
+  nsecs: 593074799
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488572.050235]: received mask_obs with header seq: 1475
+stamp:
+  secs: 1734488572
+  nsecs:  54821729
+frame_id: '' and grasped_object_mask header seq: 119185
+stamp:
+  secs: 1734488571
+  nsecs: 792914391
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488572.250317]: received mask_obs with header seq: 1476
+stamp:
+  secs: 1734488572
+  nsecs: 254938602
+frame_id: '' and grasped_object_mask header seq: 119191
+stamp:
+  secs: 1734488571
+  nsecs: 992753029
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488572.444493]: received mask_obs with header seq: 1477
+stamp:
+  secs: 1734488572
+  nsecs: 449152708
+frame_id: '' and grasped_object_mask header seq: 119197
+stamp:
+  secs: 1734488572
+  nsecs: 192591906
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488572.651705]: received mask_obs with header seq: 1478
+stamp:
+  secs: 1734488572
+  nsecs: 656347990
+frame_id: '' and grasped_object_mask header seq: 119204
+stamp:
+  secs: 1734488572
+  nsecs: 425737619
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488572.846389]: received mask_obs with header seq: 1479
+stamp:
+  secs: 1734488572
+  nsecs: 850614070
+frame_id: '' and grasped_object_mask header seq: 119209
+stamp:
+  secs: 1734488572
+  nsecs: 592269421
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488573.059858]: received mask_obs with header seq: 1480
+stamp:
+  secs: 1734488573
+  nsecs:  64097881
+frame_id: '' and grasped_object_mask header seq: 119216
+stamp:
+  secs: 1734488572
+  nsecs: 825417042
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488573.263285]: received mask_obs with header seq: 1481
+stamp:
+  secs: 1734488573
+  nsecs: 267984390
+frame_id: '' and grasped_object_mask header seq: 119222
+stamp:
+  secs: 1734488573
+  nsecs:  25257587
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488573.460361]: received mask_obs with header seq: 1482
+stamp:
+  secs: 1734488573
+  nsecs: 465042114
+frame_id: '' and grasped_object_mask header seq: 119228
+stamp:
+  secs: 1734488573
+  nsecs: 225097895
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488573.670319]: received mask_obs with header seq: 1483
+stamp:
+  secs: 1734488573
+  nsecs: 674854516
+frame_id: '' and grasped_object_mask header seq: 119234
+stamp:
+  secs: 1734488573
+  nsecs: 424938440
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488573.859655]: received mask_obs with header seq: 1484
+stamp:
+  secs: 1734488573
+  nsecs: 863938093
+frame_id: '' and grasped_object_mask header seq: 119240
+stamp:
+  secs: 1734488573
+  nsecs: 624779940
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488574.060157]: received mask_obs with header seq: 1485
+stamp:
+  secs: 1734488574
+  nsecs:  64419031
+frame_id: '' and grasped_object_mask header seq: 119246
+stamp:
+  secs: 1734488573
+  nsecs: 824620962
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488574.255778]: received mask_obs with header seq: 1486
+stamp:
+  secs: 1734488574
+  nsecs: 260196924
+frame_id: '' and grasped_object_mask header seq: 119252
+stamp:
+  secs: 1734488574
+  nsecs:  24461746
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488574.449441]: received mask_obs with header seq: 1487
+stamp:
+  secs: 1734488574
+  nsecs: 454118967
+frame_id: '' and grasped_object_mask header seq: 119257
+stamp:
+  secs: 1734488574
+  nsecs: 190996647
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488574.646457]: received mask_obs with header seq: 1488
+stamp:
+  secs: 1734488574
+  nsecs: 651059627
+frame_id: '' and grasped_object_mask header seq: 119264
+stamp:
+  secs: 1734488574
+  nsecs: 424144030
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488574.846185]: received mask_obs with header seq: 1489
+stamp:
+  secs: 1734488574
+  nsecs: 850825548
+frame_id: '' and grasped_object_mask header seq: 119270
+stamp:
+  secs: 1734488574
+  nsecs: 623985529
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488575.050954]: received mask_obs with header seq: 1490
+stamp:
+  secs: 1734488575
+  nsecs:  55735349
+frame_id: '' and grasped_object_mask header seq: 119276
+stamp:
+  secs: 1734488574
+  nsecs: 823828459
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488575.245366]: received mask_obs with header seq: 1491
+stamp:
+  secs: 1734488575
+  nsecs: 249246835
+frame_id: '' and grasped_object_mask header seq: 119282
+stamp:
+  secs: 1734488575
+  nsecs:  23670435
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488575.446708]: received mask_obs with header seq: 1492
+stamp:
+  secs: 1734488575
+  nsecs: 449606895
+frame_id: '' and grasped_object_mask header seq: 119288
+stamp:
+  secs: 1734488575
+  nsecs: 223512650
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488575.645555]: received mask_obs with header seq: 1493
+stamp:
+  secs: 1734488575
+  nsecs: 649819135
+frame_id: '' and grasped_object_mask header seq: 119293
+stamp:
+  secs: 1734488575
+  nsecs: 390048504
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488575.860207]: received mask_obs with header seq: 1494
+stamp:
+  secs: 1734488575
+  nsecs: 865018129
+frame_id: '' and grasped_object_mask header seq: 119300
+stamp:
+  secs: 1734488575
+  nsecs: 623196363
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488576.058419]: received mask_obs with header seq: 1495
+stamp:
+  secs: 1734488576
+  nsecs:  62748432
+frame_id: '' and grasped_object_mask header seq: 119306
+stamp:
+  secs: 1734488575
+  nsecs: 823033810
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488576.254868]: received mask_obs with header seq: 1496
+stamp:
+  secs: 1734488576
+  nsecs: 259683132
+frame_id: '' and grasped_object_mask header seq: 119312
+stamp:
+  secs: 1734488576
+  nsecs:  22871733
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488576.450124]: received mask_obs with header seq: 1497
+stamp:
+  secs: 1734488576
+  nsecs: 454505920
+frame_id: '' and grasped_object_mask header seq: 119318
+stamp:
+  secs: 1734488576
+  nsecs: 222708941
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488576.650063]: received mask_obs with header seq: 1498
+stamp:
+  secs: 1734488576
+  nsecs: 654848814
+frame_id: '' and grasped_object_mask header seq: 119324
+stamp:
+  secs: 1734488576
+  nsecs: 422546148
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488576.850615]: received mask_obs with header seq: 1499
+stamp:
+  secs: 1734488576
+  nsecs: 855276107
+frame_id: '' and grasped_object_mask header seq: 119330
+stamp:
+  secs: 1734488576
+  nsecs: 622383356
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488577.053985]: received mask_obs with header seq: 1500
+stamp:
+  secs: 1734488577
+  nsecs:  58812141
+frame_id: '' and grasped_object_mask header seq: 119336
+stamp:
+  secs: 1734488576
+  nsecs: 822219610
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488577.246301]: received mask_obs with header seq: 1501
+stamp:
+  secs: 1734488577
+  nsecs: 251011610
+frame_id: '' and grasped_object_mask header seq: 119342
+stamp:
+  secs: 1734488577
+  nsecs:  22056341
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488577.451205]: received mask_obs with header seq: 1502
+stamp:
+  secs: 1734488577
+  nsecs: 455959558
+frame_id: '' and grasped_object_mask header seq: 119348
+stamp:
+  secs: 1734488577
+  nsecs: 221894264
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -101924911 for nearest: 1734488577319806084 - target: 1734488577421730995 at idx 225
+[INFO] [1734488577.650209]: received mask_obs with header seq: 1503
+stamp:
+  secs: 1734488577
+  nsecs: 655004262
+frame_id: '' and grasped_object_mask header seq: 119354
+stamp:
+  secs: 1734488577
+  nsecs: 421730995
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488577.851864]: received mask_obs with header seq: 1504
+stamp:
+  secs: 1734488577
+  nsecs: 856585025
+frame_id: '' and grasped_object_mask header seq: 119360
+stamp:
+  secs: 1734488577
+  nsecs: 621568918
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488578.053150]: received mask_obs with header seq: 1505
+stamp:
+  secs: 1734488578
+  nsecs:  57719707
+frame_id: '' and grasped_object_mask header seq: 119365
+stamp:
+  secs: 1734488577
+  nsecs: 788103819
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488578.249491]: received mask_obs with header seq: 1506
+stamp:
+  secs: 1734488578
+  nsecs: 254211425
+frame_id: '' and grasped_object_mask header seq: 119372
+stamp:
+  secs: 1734488578
+  nsecs:  21251440
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488578.460625]: received mask_obs with header seq: 1507
+stamp:
+  secs: 1734488578
+  nsecs: 465170621
+frame_id: '' and grasped_object_mask header seq: 119378
+stamp:
+  secs: 1734488578
+  nsecs: 221092701
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488578.661480]: received mask_obs with header seq: 1508
+stamp:
+  secs: 1734488578
+  nsecs: 666278839
+frame_id: '' and grasped_object_mask header seq: 119384
+stamp:
+  secs: 1734488578
+  nsecs: 420935392
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488578.869227]: received mask_obs with header seq: 1509
+stamp:
+  secs: 1734488578
+  nsecs: 874022722
+frame_id: '' and grasped_object_mask header seq: 119390
+stamp:
+  secs: 1734488578
+  nsecs: 620775223
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488579.072507]: received mask_obs with header seq: 1510
+stamp:
+  secs: 1734488579
+  nsecs:  77324628
+frame_id: '' and grasped_object_mask header seq: 119396
+stamp:
+  secs: 1734488578
+  nsecs: 820610046
+frame_id: "camera_color_optical_frame"
+preempting episode
+[INFO] [1734488579.137152]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1734488579.267859]: received mask_obs with header seq: 1511
+stamp:
+  secs: 1734488579
+  nsecs: 272503852
+frame_id: '' and grasped_object_mask header seq: 119402
+stamp:
+  secs: 1734488579
+  nsecs:  20444870
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488579.851763]: Stored episode 2.
+[INFO] [1734488579.852542]: Saving video...
+[INFO] [1734488580.370419]: Video saved!
+Starting episode 2
+[INFO] [1734488580.413283]: resetting environment
+[INFO] [1734488580.423526]: cleared current plan
+[INFO] [1734488580.423770]: moving to home
+[INFO] [1734488583.831673]: reached home
+[INFO] [1734488583.834441]: reset action history
+[INFO] [1734488586.602740]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734488600.975153]: resetting environment
+[INFO] [1734488600.983774]: cleared current plan
+[INFO] [1734488600.985259]: moving to home
+[INFO] [1734488601.987035]: reached home
+[INFO] [1734488601.987929]: reset action history
+[INFO] [1734488604.695550]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/episode_rosbags/episode_2_2024-12-17-21-23-24.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-76272.log
+started roslaunch server http://158.130.50.37:45511/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [76851]
+process[print_text-8]: started with pid [76852]
+process[rosbag_record-9]: started with pid [76853]
+started bagging!
+[INFO] [1734488605.362452]: received mask_obs with header seq: 1512
+stamp:
+  secs: 1734488605
+  nsecs: 366941928
+frame_id: '' and grasped_object_mask header seq: 120187
+stamp:
+  secs: 1734488605
+  nsecs: 174066544
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488605.571699]: received mask_obs with header seq: 1513
+stamp:
+  secs: 1734488605
+  nsecs: 576406478
+frame_id: '' and grasped_object_mask header seq: 120193
+stamp:
+  secs: 1734488605
+  nsecs: 373648882
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488605.768146]: received mask_obs with header seq: 1514
+stamp:
+  secs: 1734488605
+  nsecs: 772837877
+frame_id: '' and grasped_object_mask header seq: 120199
+stamp:
+  secs: 1734488605
+  nsecs: 573219299
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488605.974410]: received mask_obs with header seq: 1515
+stamp:
+  secs: 1734488605
+  nsecs: 979129314
+frame_id: '' and grasped_object_mask header seq: 120205
+stamp:
+  secs: 1734488605
+  nsecs: 772812843
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488606.177917]: received mask_obs with header seq: 1516
+stamp:
+  secs: 1734488606
+  nsecs: 182721614
+frame_id: '' and grasped_object_mask header seq: 120211
+stamp:
+  secs: 1734488605
+  nsecs: 972412109
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488606.383939]: received mask_obs with header seq: 1517
+stamp:
+  secs: 1734488606
+  nsecs: 388672828
+frame_id: '' and grasped_object_mask header seq: 120217
+stamp:
+  secs: 1734488606
+  nsecs: 172000408
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488606.587792]: received mask_obs with header seq: 1518
+stamp:
+  secs: 1734488606
+  nsecs: 592413663
+frame_id: '' and grasped_object_mask header seq: 120223
+stamp:
+  secs: 1734488606
+  nsecs: 371575832
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488606.791047]: received mask_obs with header seq: 1519
+stamp:
+  secs: 1734488606
+  nsecs: 795864582
+frame_id: '' and grasped_object_mask header seq: 120229
+stamp:
+  secs: 1734488606
+  nsecs: 571139574
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488606.992195]: received mask_obs with header seq: 1520
+stamp:
+  secs: 1734488606
+  nsecs: 996987342
+frame_id: '' and grasped_object_mask header seq: 120235
+stamp:
+  secs: 1734488606
+  nsecs: 770735264
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488607.197996]: received mask_obs with header seq: 1521
+stamp:
+  secs: 1734488607
+  nsecs: 202747583
+frame_id: '' and grasped_object_mask header seq: 120241
+stamp:
+  secs: 1734488606
+  nsecs: 970341206
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488607.394931]: received mask_obs with header seq: 1522
+stamp:
+  secs: 1734488607
+  nsecs: 399050951
+frame_id: '' and grasped_object_mask header seq: 120247
+stamp:
+  secs: 1734488607
+  nsecs: 169935703
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488607.595042]: received mask_obs with header seq: 1523
+stamp:
+  secs: 1734488607
+  nsecs: 599728345
+frame_id: '' and grasped_object_mask header seq: 120253
+stamp:
+  secs: 1734488607
+  nsecs: 369518280
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488607.792655]: received mask_obs with header seq: 1524
+stamp:
+  secs: 1734488607
+  nsecs: 797485351
+frame_id: '' and grasped_object_mask header seq: 120259
+stamp:
+  secs: 1734488607
+  nsecs: 569090366
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488607.994668]: received mask_obs with header seq: 1525
+stamp:
+  secs: 1734488607
+  nsecs: 999407768
+frame_id: '' and grasped_object_mask header seq: 120265
+stamp:
+  secs: 1734488607
+  nsecs: 768693924
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488608.193706]: received mask_obs with header seq: 1526
+stamp:
+  secs: 1734488608
+  nsecs: 198462247
+frame_id: '' and grasped_object_mask header seq: 120271
+stamp:
+  secs: 1734488607
+  nsecs: 968308449
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488608.392981]: received mask_obs with header seq: 1527
+stamp:
+  secs: 1734488608
+  nsecs: 397598981
+frame_id: '' and grasped_object_mask header seq: 120277
+stamp:
+  secs: 1734488608
+  nsecs: 167911530
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488608.593791]: received mask_obs with header seq: 1528
+stamp:
+  secs: 1734488608
+  nsecs: 598575830
+frame_id: '' and grasped_object_mask header seq: 120283
+stamp:
+  secs: 1734488608
+  nsecs: 367503405
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488608.793671]: received mask_obs with header seq: 1529
+stamp:
+  secs: 1734488608
+  nsecs: 798375368
+frame_id: '' and grasped_object_mask header seq: 120289
+stamp:
+  secs: 1734488608
+  nsecs: 567084312
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488608.993589]: received mask_obs with header seq: 1530
+stamp:
+  secs: 1734488608
+  nsecs: 998181343
+frame_id: '' and grasped_object_mask header seq: 120295
+stamp:
+  secs: 1734488608
+  nsecs: 766689301
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488609.193192]: received mask_obs with header seq: 1531
+stamp:
+  secs: 1734488609
+  nsecs: 197862625
+frame_id: '' and grasped_object_mask header seq: 120301
+stamp:
+  secs: 1734488608
+  nsecs: 966298819
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488609.392704]: received mask_obs with header seq: 1532
+stamp:
+  secs: 1734488609
+  nsecs: 397487878
+frame_id: '' and grasped_object_mask header seq: 120307
+stamp:
+  secs: 1734488609
+  nsecs: 165896893
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488609.592861]: received mask_obs with header seq: 1533
+stamp:
+  secs: 1734488609
+  nsecs: 597646951
+frame_id: '' and grasped_object_mask header seq: 120313
+stamp:
+  secs: 1734488609
+  nsecs: 365483761
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488609.792755]: received mask_obs with header seq: 1534
+stamp:
+  secs: 1734488609
+  nsecs: 797553300
+frame_id: '' and grasped_object_mask header seq: 120319
+stamp:
+  secs: 1734488609
+  nsecs: 565058947
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488609.995408]: received mask_obs with header seq: 1535
+stamp:
+  secs: 1734488610
+  nsecs:     63419
+frame_id: '' and grasped_object_mask header seq: 120325
+stamp:
+  secs: 1734488609
+  nsecs: 764657974
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488610.192667]: received mask_obs with header seq: 1536
+stamp:
+  secs: 1734488610
+  nsecs: 197435617
+frame_id: '' and grasped_object_mask header seq: 120331
+stamp:
+  secs: 1734488609
+  nsecs: 964264393
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488610.393017]: received mask_obs with header seq: 1537
+stamp:
+  secs: 1734488610
+  nsecs: 397775173
+frame_id: '' and grasped_object_mask header seq: 120337
+stamp:
+  secs: 1734488610
+  nsecs: 163858414
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488610.603862]: received mask_obs with header seq: 1538
+stamp:
+  secs: 1734488610
+  nsecs: 608621120
+frame_id: '' and grasped_object_mask header seq: 120343
+stamp:
+  secs: 1734488610
+  nsecs: 363440514
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488610.814958]: received mask_obs with header seq: 1539
+stamp:
+  secs: 1734488610
+  nsecs: 819467306
+frame_id: '' and grasped_object_mask header seq: 120350
+stamp:
+  secs: 1734488610
+  nsecs: 596272230
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488611.004504]: received mask_obs with header seq: 1540
+stamp:
+  secs: 1734488611
+  nsecs:   8985519
+frame_id: '' and grasped_object_mask header seq: 120355
+stamp:
+  secs: 1734488610
+  nsecs: 762609005
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488611.199563]: received mask_obs with header seq: 1541
+stamp:
+  secs: 1734488611
+  nsecs: 204004526
+frame_id: '' and grasped_object_mask header seq: 120361
+stamp:
+  secs: 1734488610
+  nsecs: 962214708
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488611.391644]: received mask_obs with header seq: 1542
+stamp:
+  secs: 1734488611
+  nsecs: 396319389
+frame_id: '' and grasped_object_mask header seq: 120367
+stamp:
+  secs: 1734488611
+  nsecs: 161809683
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488611.591723]: received mask_obs with header seq: 1543
+stamp:
+  secs: 1734488611
+  nsecs: 596360206
+frame_id: '' and grasped_object_mask header seq: 120373
+stamp:
+  secs: 1734488611
+  nsecs: 361391783
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488611.791330]: received mask_obs with header seq: 1544
+stamp:
+  secs: 1734488611
+  nsecs: 795974969
+frame_id: '' and grasped_object_mask header seq: 120379
+stamp:
+  secs: 1734488611
+  nsecs: 560962439
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488611.992624]: received mask_obs with header seq: 1545
+stamp:
+  secs: 1734488611
+  nsecs: 997135162
+frame_id: '' and grasped_object_mask header seq: 120385
+stamp:
+  secs: 1734488611
+  nsecs: 760560989
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488612.193075]: received mask_obs with header seq: 1546
+stamp:
+  secs: 1734488612
+  nsecs: 197808742
+frame_id: '' and grasped_object_mask header seq: 120391
+stamp:
+  secs: 1734488611
+  nsecs: 960167646
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488612.392593]: received mask_obs with header seq: 1547
+stamp:
+  secs: 1734488612
+  nsecs: 397080659
+frame_id: '' and grasped_object_mask header seq: 120397
+stamp:
+  secs: 1734488612
+  nsecs: 159762621
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488612.591845]: received mask_obs with header seq: 1548
+stamp:
+  secs: 1734488612
+  nsecs: 596584081
+frame_id: '' and grasped_object_mask header seq: 120403
+stamp:
+  secs: 1734488612
+  nsecs: 359346867
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488612.791497]: received mask_obs with header seq: 1549
+stamp:
+  secs: 1734488612
+  nsecs: 795956850
+frame_id: '' and grasped_object_mask header seq: 120409
+stamp:
+  secs: 1734488612
+  nsecs: 558918476
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488612.991452]: received mask_obs with header seq: 1550
+stamp:
+  secs: 1734488612
+  nsecs: 995924949
+frame_id: '' and grasped_object_mask header seq: 120415
+stamp:
+  secs: 1734488612
+  nsecs: 758515596
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488613.191323]: received mask_obs with header seq: 1551
+stamp:
+  secs: 1734488613
+  nsecs: 195973157
+frame_id: '' and grasped_object_mask header seq: 120421
+stamp:
+  secs: 1734488612
+  nsecs: 958119154
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488613.392769]: received mask_obs with header seq: 1552
+stamp:
+  secs: 1734488613
+  nsecs: 397516250
+frame_id: '' and grasped_object_mask header seq: 120427
+stamp:
+  secs: 1734488613
+  nsecs: 157711029
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488613.600884]: received mask_obs with header seq: 1553
+stamp:
+  secs: 1734488613
+  nsecs: 605432987
+frame_id: '' and grasped_object_mask header seq: 120433
+stamp:
+  secs: 1734488613
+  nsecs: 357290745
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488613.816622]: received mask_obs with header seq: 1554
+stamp:
+  secs: 1734488613
+  nsecs: 821230411
+frame_id: '' and grasped_object_mask header seq: 120439
+stamp:
+  secs: 1734488613
+  nsecs: 556859970
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488614.011085]: received mask_obs with header seq: 1555
+stamp:
+  secs: 1734488614
+  nsecs:  15702962
+frame_id: '' and grasped_object_mask header seq: 120445
+stamp:
+  secs: 1734488613
+  nsecs: 756456137
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488614.200415]: received mask_obs with header seq: 1556
+stamp:
+  secs: 1734488614
+  nsecs: 205089569
+frame_id: '' and grasped_object_mask header seq: 120451
+stamp:
+  secs: 1734488613
+  nsecs: 956060171
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488614.394276]: received mask_obs with header seq: 1557
+stamp:
+  secs: 1734488614
+  nsecs: 398661136
+frame_id: '' and grasped_object_mask header seq: 120457
+stamp:
+  secs: 1734488614
+  nsecs: 155652285
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488614.592327]: received mask_obs with header seq: 1558
+stamp:
+  secs: 1734488614
+  nsecs: 597019910
+frame_id: '' and grasped_object_mask header seq: 120463
+stamp:
+  secs: 1734488614
+  nsecs: 355232954
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488614.792209]: received mask_obs with header seq: 1559
+stamp:
+  secs: 1734488614
+  nsecs: 796582460
+frame_id: '' and grasped_object_mask header seq: 120469
+stamp:
+  secs: 1734488614
+  nsecs: 554801464
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488614.993536]: received mask_obs with header seq: 1560
+stamp:
+  secs: 1734488614
+  nsecs: 998095512
+frame_id: '' and grasped_object_mask header seq: 120475
+stamp:
+  secs: 1734488614
+  nsecs: 754396677
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488615.193352]: received mask_obs with header seq: 1561
+stamp:
+  secs: 1734488615
+  nsecs: 198084354
+frame_id: '' and grasped_object_mask header seq: 120481
+stamp:
+  secs: 1734488614
+  nsecs: 953999996
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488615.393521]: received mask_obs with header seq: 1562
+stamp:
+  secs: 1734488615
+  nsecs: 398257493
+frame_id: '' and grasped_object_mask header seq: 120487
+stamp:
+  secs: 1734488615
+  nsecs: 153590441
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488615.591379]: received mask_obs with header seq: 1563
+stamp:
+  secs: 1734488615
+  nsecs: 596136331
+frame_id: '' and grasped_object_mask header seq: 120493
+stamp:
+  secs: 1734488615
+  nsecs: 353168964
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488615.793644]: received mask_obs with header seq: 1564
+stamp:
+  secs: 1734488615
+  nsecs: 798228263
+frame_id: '' and grasped_object_mask header seq: 120499
+stamp:
+  secs: 1734488615
+  nsecs: 552735567
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488615.993771]: received mask_obs with header seq: 1565
+stamp:
+  secs: 1734488615
+  nsecs: 998428106
+frame_id: '' and grasped_object_mask header seq: 120505
+stamp:
+  secs: 1734488615
+  nsecs: 752336502
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488616.193889]: received mask_obs with header seq: 1566
+stamp:
+  secs: 1734488616
+  nsecs: 198642253
+frame_id: '' and grasped_object_mask header seq: 120511
+stamp:
+  secs: 1734488615
+  nsecs: 951948643
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488616.393542]: received mask_obs with header seq: 1567
+stamp:
+  secs: 1734488616
+  nsecs: 398267984
+frame_id: '' and grasped_object_mask header seq: 120517
+stamp:
+  secs: 1734488616
+  nsecs: 151550531
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488616.593372]: received mask_obs with header seq: 1568
+stamp:
+  secs: 1734488616
+  nsecs: 597777128
+frame_id: '' and grasped_object_mask header seq: 120523
+stamp:
+  secs: 1734488616
+  nsecs: 351139784
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488616.806933]: received mask_obs with header seq: 1569
+stamp:
+  secs: 1734488616
+  nsecs: 811230421
+frame_id: '' and grasped_object_mask header seq: 120529
+stamp:
+  secs: 1734488616
+  nsecs: 550718069
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488617.015355]: received mask_obs with header seq: 1570
+stamp:
+  secs: 1734488617
+  nsecs:  19805908
+frame_id: '' and grasped_object_mask header seq: 120535
+stamp:
+  secs: 1734488616
+  nsecs: 751157999
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488617.216903]: received mask_obs with header seq: 1571
+stamp:
+  secs: 1734488617
+  nsecs: 221486568
+frame_id: '' and grasped_object_mask header seq: 120541
+stamp:
+  secs: 1734488616
+  nsecs: 952105284
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488617.420572]: received mask_obs with header seq: 1572
+stamp:
+  secs: 1734488617
+  nsecs: 425257205
+frame_id: '' and grasped_object_mask header seq: 120548
+stamp:
+  secs: 1734488617
+  nsecs: 186654091
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488617.620439]: received mask_obs with header seq: 1573
+stamp:
+  secs: 1734488617
+  nsecs: 624833583
+frame_id: '' and grasped_object_mask header seq: 120554
+stamp:
+  secs: 1734488617
+  nsecs: 387789488
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488617.815166]: received mask_obs with header seq: 1574
+stamp:
+  secs: 1734488617
+  nsecs: 819962024
+frame_id: '' and grasped_object_mask header seq: 120560
+stamp:
+  secs: 1734488617
+  nsecs: 589011669
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488618.014327]: received mask_obs with header seq: 1575
+stamp:
+  secs: 1734488618
+  nsecs:  18929243
+frame_id: '' and grasped_object_mask header seq: 120566
+stamp:
+  secs: 1734488617
+  nsecs: 789096355
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488618.212758]: received mask_obs with header seq: 1576
+stamp:
+  secs: 1734488618
+  nsecs: 217058658
+frame_id: '' and grasped_object_mask header seq: 120572
+stamp:
+  secs: 1734488617
+  nsecs: 988945007
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488618.404349]: received mask_obs with header seq: 1577
+stamp:
+  secs: 1734488618
+  nsecs: 408903121
+frame_id: '' and grasped_object_mask header seq: 120577
+stamp:
+  secs: 1734488618
+  nsecs: 155484676
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488618.594722]: received mask_obs with header seq: 1578
+stamp:
+  secs: 1734488618
+  nsecs: 599435806
+frame_id: '' and grasped_object_mask header seq: 120583
+stamp:
+  secs: 1734488618
+  nsecs: 355333567
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488618.794130]: received mask_obs with header seq: 1579
+stamp:
+  secs: 1734488618
+  nsecs: 798756361
+frame_id: '' and grasped_object_mask header seq: 120589
+stamp:
+  secs: 1734488618
+  nsecs: 555183887
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488618.994119]: received mask_obs with header seq: 1580
+stamp:
+  secs: 1734488618
+  nsecs: 998494863
+frame_id: '' and grasped_object_mask header seq: 120595
+stamp:
+  secs: 1734488618
+  nsecs: 755027771
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488619.193269]: received mask_obs with header seq: 1581
+stamp:
+  secs: 1734488619
+  nsecs: 198017835
+frame_id: '' and grasped_object_mask header seq: 120601
+stamp:
+  secs: 1734488618
+  nsecs: 954868793
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488619.394068]: received mask_obs with header seq: 1582
+stamp:
+  secs: 1734488619
+  nsecs: 398792028
+frame_id: '' and grasped_object_mask header seq: 120607
+stamp:
+  secs: 1734488619
+  nsecs: 154710293
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488619.594548]: received mask_obs with header seq: 1583
+stamp:
+  secs: 1734488619
+  nsecs: 599262952
+frame_id: '' and grasped_object_mask header seq: 120613
+stamp:
+  secs: 1734488619
+  nsecs: 354551554
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488619.806545]: received mask_obs with header seq: 1584
+stamp:
+  secs: 1734488619
+  nsecs: 810845851
+frame_id: '' and grasped_object_mask header seq: 120619
+stamp:
+  secs: 1734488619
+  nsecs: 554392815
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488619.998449]: received mask_obs with header seq: 1585
+stamp:
+  secs: 1734488620
+  nsecs:   2909898
+frame_id: '' and grasped_object_mask header seq: 120625
+stamp:
+  secs: 1734488619
+  nsecs: 754235744
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488620.194699]: received mask_obs with header seq: 1586
+stamp:
+  secs: 1734488620
+  nsecs: 199217081
+frame_id: '' and grasped_object_mask header seq: 120631
+stamp:
+  secs: 1734488619
+  nsecs: 954078197
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488620.395786]: received mask_obs with header seq: 1587
+stamp:
+  secs: 1734488620
+  nsecs: 400550842
+frame_id: '' and grasped_object_mask header seq: 120637
+stamp:
+  secs: 1734488620
+  nsecs: 153920412
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488620.594914]: received mask_obs with header seq: 1588
+stamp:
+  secs: 1734488620
+  nsecs: 599648237
+frame_id: '' and grasped_object_mask header seq: 120643
+stamp:
+  secs: 1734488620
+  nsecs: 353762865
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488620.795459]: received mask_obs with header seq: 1589
+stamp:
+  secs: 1734488620
+  nsecs: 800186157
+frame_id: '' and grasped_object_mask header seq: 120649
+stamp:
+  secs: 1734488620
+  nsecs: 553605556
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488620.995793]: received mask_obs with header seq: 1590
+stamp:
+  secs: 1734488621
+  nsecs:    473260
+frame_id: '' and grasped_object_mask header seq: 120655
+stamp:
+  secs: 1734488620
+  nsecs: 753448009
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488621.196666]: received mask_obs with header seq: 1591
+stamp:
+  secs: 1734488621
+  nsecs: 201425552
+frame_id: '' and grasped_object_mask header seq: 120661
+stamp:
+  secs: 1734488620
+  nsecs: 953291178
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488621.393859]: received mask_obs with header seq: 1592
+stamp:
+  secs: 1734488621
+  nsecs: 398572206
+frame_id: '' and grasped_object_mask header seq: 120667
+stamp:
+  secs: 1734488621
+  nsecs: 153134108
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488621.595337]: received mask_obs with header seq: 1593
+stamp:
+  secs: 1734488621
+  nsecs: 599834918
+frame_id: '' and grasped_object_mask header seq: 120673
+stamp:
+  secs: 1734488621
+  nsecs: 352976799
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488621.796055]: received mask_obs with header seq: 1594
+stamp:
+  secs: 1734488621
+  nsecs: 800581693
+frame_id: '' and grasped_object_mask header seq: 120679
+stamp:
+  secs: 1734488621
+  nsecs: 552819729
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488621.997539]: received mask_obs with header seq: 1595
+stamp:
+  secs: 1734488622
+  nsecs:   1941680
+frame_id: '' and grasped_object_mask header seq: 120685
+stamp:
+  secs: 1734488621
+  nsecs: 752654314
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488622.213064]: received mask_obs with header seq: 1596
+stamp:
+  secs: 1734488622
+  nsecs: 217785358
+frame_id: '' and grasped_object_mask header seq: 120691
+stamp:
+  secs: 1734488621
+  nsecs: 952483654
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488622.412414]: received mask_obs with header seq: 1597
+stamp:
+  secs: 1734488622
+  nsecs: 417165279
+frame_id: '' and grasped_object_mask header seq: 120698
+stamp:
+  secs: 1734488622
+  nsecs: 185618162
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488622.597838]: received mask_obs with header seq: 1598
+stamp:
+  secs: 1734488622
+  nsecs: 602610349
+frame_id: '' and grasped_object_mask header seq: 120703
+stamp:
+  secs: 1734488622
+  nsecs: 352142334
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488622.796506]: received mask_obs with header seq: 1599
+stamp:
+  secs: 1734488622
+  nsecs: 801192522
+frame_id: '' and grasped_object_mask header seq: 120709
+stamp:
+  secs: 1734488622
+  nsecs: 551970482
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488623.015378]: received mask_obs with header seq: 1600
+stamp:
+  secs: 1734488623
+  nsecs:  20085573
+frame_id: '' and grasped_object_mask header seq: 120715
+stamp:
+  secs: 1734488622
+  nsecs: 751805067
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488623.213866]: received mask_obs with header seq: 1601
+stamp:
+  secs: 1734488623
+  nsecs: 218625068
+frame_id: '' and grasped_object_mask header seq: 120722
+stamp:
+  secs: 1734488622
+  nsecs: 984950304
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488623.408698]: received mask_obs with header seq: 1602
+stamp:
+  secs: 1734488623
+  nsecs: 413304090
+frame_id: '' and grasped_object_mask header seq: 120728
+stamp:
+  secs: 1734488623
+  nsecs: 184788227
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488623.600203]: received mask_obs with header seq: 1603
+stamp:
+  secs: 1734488623
+  nsecs: 604870796
+frame_id: '' and grasped_object_mask header seq: 120734
+stamp:
+  secs: 1734488623
+  nsecs: 384626389
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488623.813736]: received mask_obs with header seq: 1604
+stamp:
+  secs: 1734488623
+  nsecs: 818533658
+frame_id: '' and grasped_object_mask header seq: 120739
+stamp:
+  secs: 1734488623
+  nsecs: 551158905
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488624.014790]: received mask_obs with header seq: 1605
+stamp:
+  secs: 1734488624
+  nsecs:  19541263
+frame_id: '' and grasped_object_mask header seq: 120746
+stamp:
+  secs: 1734488623
+  nsecs: 784304142
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488624.212339]: received mask_obs with header seq: 1606
+stamp:
+  secs: 1734488624
+  nsecs: 217118501
+frame_id: '' and grasped_object_mask header seq: 120752
+stamp:
+  secs: 1734488623
+  nsecs: 984144211
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488624.410587]: received mask_obs with header seq: 1607
+stamp:
+  secs: 1734488624
+  nsecs: 415372610
+frame_id: '' and grasped_object_mask header seq: 120758
+stamp:
+  secs: 1734488624
+  nsecs: 183985233
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488624.609923]: received mask_obs with header seq: 1608
+stamp:
+  secs: 1734488624
+  nsecs: 614696025
+frame_id: '' and grasped_object_mask header seq: 120764
+stamp:
+  secs: 1734488624
+  nsecs: 383825779
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488624.802581]: received mask_obs with header seq: 1609
+stamp:
+  secs: 1734488624
+  nsecs: 807047843
+frame_id: '' and grasped_object_mask header seq: 120770
+stamp:
+  secs: 1734488624
+  nsecs: 583666325
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488624.994152]: received mask_obs with header seq: 1610
+stamp:
+  secs: 1734488624
+  nsecs: 998709440
+frame_id: '' and grasped_object_mask header seq: 120775
+stamp:
+  secs: 1734488624
+  nsecs: 750202179
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488625.194065]: received mask_obs with header seq: 1611
+stamp:
+  secs: 1734488625
+  nsecs: 198657512
+frame_id: '' and grasped_object_mask header seq: 120781
+stamp:
+  secs: 1734488624
+  nsecs: 950044870
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488625.394781]: received mask_obs with header seq: 1612
+stamp:
+  secs: 1734488625
+  nsecs: 399248838
+frame_id: '' and grasped_object_mask header seq: 120787
+stamp:
+  secs: 1734488625
+  nsecs: 149888277
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488625.594160]: received mask_obs with header seq: 1613
+stamp:
+  secs: 1734488625
+  nsecs: 598845720
+frame_id: '' and grasped_object_mask header seq: 120793
+stamp:
+  secs: 1734488625
+  nsecs: 349731684
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488625.811262]: received mask_obs with header seq: 1614
+stamp:
+  secs: 1734488625
+  nsecs: 816011667
+frame_id: '' and grasped_object_mask header seq: 120799
+stamp:
+  secs: 1734488625
+  nsecs: 549575090
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488626.012267]: received mask_obs with header seq: 1615
+stamp:
+  secs: 1734488626
+  nsecs:  16935825
+frame_id: '' and grasped_object_mask header seq: 120806
+stamp:
+  secs: 1734488625
+  nsecs: 782722473
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488626.205694]: received mask_obs with header seq: 1616
+stamp:
+  secs: 1734488626
+  nsecs: 210468769
+frame_id: '' and grasped_object_mask header seq: 120812
+stamp:
+  secs: 1734488625
+  nsecs: 982560635
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488626.405659]: received mask_obs with header seq: 1617
+stamp:
+  secs: 1734488626
+  nsecs: 409345626
+frame_id: '' and grasped_object_mask header seq: 120817
+stamp:
+  secs: 1734488626
+  nsecs: 149092913
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488626.620202]: received mask_obs with header seq: 1618
+stamp:
+  secs: 1734488626
+  nsecs: 624992847
+frame_id: '' and grasped_object_mask header seq: 120824
+stamp:
+  secs: 1734488626
+  nsecs: 382236958
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488626.815510]: received mask_obs with header seq: 1619
+stamp:
+  secs: 1734488626
+  nsecs: 820239305
+frame_id: '' and grasped_object_mask header seq: 120830
+stamp:
+  secs: 1734488626
+  nsecs: 582076073
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488627.011742]: received mask_obs with header seq: 1620
+stamp:
+  secs: 1734488627
+  nsecs:  16483783
+frame_id: '' and grasped_object_mask header seq: 120836
+stamp:
+  secs: 1734488626
+  nsecs: 781913757
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -261975177 for nearest: 1734488626719776027 - target: 1734488626981751204 at idx 249
+[INFO] [1734488627.210031]: received mask_obs with header seq: 1621
+stamp:
+  secs: 1734488627
+  nsecs: 214428663
+frame_id: '' and grasped_object_mask header seq: 120842
+stamp:
+  secs: 1734488626
+  nsecs: 981751204
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -461812384 for nearest: 1734488626719776027 - target: 1734488627181588411 at idx 249
+[INFO] [1734488627.408113]: received mask_obs with header seq: 1622
+stamp:
+  secs: 1734488627
+  nsecs: 412234306
+frame_id: '' and grasped_object_mask header seq: 120848
+stamp:
+  secs: 1734488627
+  nsecs: 181588411
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -661649831 for nearest: 1734488626719776027 - target: 1734488627381425858 at idx 249
+[INFO] [1734488627.602649]: received mask_obs with header seq: 1623
+stamp:
+  secs: 1734488627
+  nsecs: 606830120
+frame_id: '' and grasped_object_mask header seq: 120854
+stamp:
+  secs: 1734488627
+  nsecs: 381425858
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -861486562 for nearest: 1734488626719776027 - target: 1734488627581262589 at idx 249
+[INFO] [1734488627.797651]: received mask_obs with header seq: 1624
+stamp:
+  secs: 1734488627
+  nsecs: 802087783
+frame_id: '' and grasped_object_mask header seq: 120859
+stamp:
+  secs: 1734488627
+  nsecs: 547956944
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -1061324961 for nearest: 1734488626719776027 - target: 1734488627781100988 at idx 249
+[INFO] [1734488628.002376]: received mask_obs with header seq: 1625
+stamp:
+  secs: 1734488628
+  nsecs:   7000923
+frame_id: '' and grasped_object_mask header seq: 120865
+stamp:
+  secs: 1734488627
+  nsecs: 747794390
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -1261161931 for nearest: 1734488626719776027 - target: 1734488627980937958 at idx 249
+[INFO] [1734488628.197935]: received mask_obs with header seq: 1626
+stamp:
+  secs: 1734488628
+  nsecs: 202474594
+frame_id: '' and grasped_object_mask header seq: 120871
+stamp:
+  secs: 1734488627
+  nsecs: 947631359
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -1460998900 for nearest: 1734488626719776027 - target: 1734488628180774927 at idx 249
+[INFO] [1734488628.397832]: received mask_obs with header seq: 1627
+stamp:
+  secs: 1734488628
+  nsecs: 401948928
+frame_id: '' and grasped_object_mask header seq: 120877
+stamp:
+  secs: 1734488628
+  nsecs: 147468328
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -1660835870 for nearest: 1734488626719776027 - target: 1734488628380611897 at idx 249
+[INFO] [1734488628.597165]: received mask_obs with header seq: 1628
+stamp:
+  secs: 1734488628
+  nsecs: 601340055
+frame_id: '' and grasped_object_mask header seq: 120883
+stamp:
+  secs: 1734488628
+  nsecs: 347305059
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -1860672600 for nearest: 1734488626719776027 - target: 1734488628580448627 at idx 249
+[INFO] [1734488628.801036]: received mask_obs with header seq: 1629
+stamp:
+  secs: 1734488628
+  nsecs: 805765151
+frame_id: '' and grasped_object_mask header seq: 120890
+stamp:
+  secs: 1734488628
+  nsecs: 580448627
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -2027202733 for nearest: 1734488626719776027 - target: 1734488628746978760 at idx 249
+[INFO] [1734488629.006799]: received mask_obs with header seq: 1630
+stamp:
+  secs: 1734488629
+  nsecs:  11545181
+frame_id: '' and grasped_object_mask header seq: 120895
+stamp:
+  secs: 1734488628
+  nsecs: 746978760
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -2260343440 for nearest: 1734488626719776027 - target: 1734488628980119467 at idx 249
+[INFO] [1734488629.210676]: received mask_obs with header seq: 1631
+stamp:
+  secs: 1734488629
+  nsecs: 215385913
+frame_id: '' and grasped_object_mask header seq: 120902
+stamp:
+  secs: 1734488628
+  nsecs: 980119467
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -2460177548 for nearest: 1734488626719776027 - target: 1734488629179953575 at idx 249
+[INFO] [1734488629.407640]: received mask_obs with header seq: 1632
+stamp:
+  secs: 1734488629
+  nsecs: 412182569
+frame_id: '' and grasped_object_mask header seq: 120908
+stamp:
+  secs: 1734488629
+  nsecs: 179953575
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -2660011656 for nearest: 1734488626719776027 - target: 1734488629379787683 at idx 249
+[INFO] [1734488629.607066]: received mask_obs with header seq: 1633
+stamp:
+  secs: 1734488629
+  nsecs: 611466646
+frame_id: '' and grasped_object_mask header seq: 120914
+stamp:
+  secs: 1734488629
+  nsecs: 379787683
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -2859845288 for nearest: 1734488626719776027 - target: 1734488629579621315 at idx 249
+[INFO] [1734488629.805089]: received mask_obs with header seq: 1634
+stamp:
+  secs: 1734488629
+  nsecs: 809862852
+frame_id: '' and grasped_object_mask header seq: 120920
+stamp:
+  secs: 1734488629
+  nsecs: 579621315
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -3059683688 for nearest: 1734488626719776027 - target: 1734488629779459715 at idx 249
+[INFO] [1734488629.996650]: received mask_obs with header seq: 1635
+stamp:
+  secs: 1734488630
+  nsecs:   1130819
+frame_id: '' and grasped_object_mask header seq: 120925
+stamp:
+  secs: 1734488629
+  nsecs: 746153355
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -3259524710 for nearest: 1734488626719776027 - target: 1734488629979300737 at idx 249
+[INFO] [1734488630.197491]: received mask_obs with header seq: 1636
+stamp:
+  secs: 1734488630
+  nsecs: 202248811
+frame_id: '' and grasped_object_mask header seq: 120931
+stamp:
+  secs: 1734488629
+  nsecs: 945994377
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -3459365971 for nearest: 1734488626719776027 - target: 1734488630179141998 at idx 249
+[INFO] [1734488630.397286]: received mask_obs with header seq: 1637
+stamp:
+  secs: 1734488630
+  nsecs: 402032613
+frame_id: '' and grasped_object_mask header seq: 120937
+stamp:
+  secs: 1734488630
+  nsecs: 145835638
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -3659208424 for nearest: 1734488626719776027 - target: 1734488630378984451 at idx 249
+[INFO] [1734488630.597954]: received mask_obs with header seq: 1638
+stamp:
+  secs: 1734488630
+  nsecs: 602582931
+frame_id: '' and grasped_object_mask header seq: 120943
+stamp:
+  secs: 1734488630
+  nsecs: 345676899
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -3859049685 for nearest: 1734488626719776027 - target: 1734488630578825712 at idx 249
+[INFO] [1734488630.797328]: received mask_obs with header seq: 1639
+stamp:
+  secs: 1734488630
+  nsecs: 801355361
+frame_id: '' and grasped_object_mask header seq: 120949
+stamp:
+  secs: 1734488630
+  nsecs: 545518398
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -4058887370 for nearest: 1734488626719776027 - target: 1734488630778663397 at idx 249
+[INFO] [1734488630.996526]: received mask_obs with header seq: 1640
+stamp:
+  secs: 1734488631
+  nsecs:    705718
+frame_id: '' and grasped_object_mask header seq: 120956
+stamp:
+  secs: 1734488630
+  nsecs: 778663397
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -4258722909 for nearest: 1734488626719776027 - target: 1734488630978498936 at idx 249
+[INFO] [1734488631.203860]: received mask_obs with header seq: 1641
+stamp:
+  secs: 1734488631
+  nsecs: 208388328
+frame_id: '' and grasped_object_mask header seq: 120961
+stamp:
+  secs: 1734488630
+  nsecs: 945193529
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -4458558448 for nearest: 1734488626719776027 - target: 1734488631178334475 at idx 249
+[INFO] [1734488631.397352]: received mask_obs with header seq: 1642
+stamp:
+  secs: 1734488631
+  nsecs: 402099132
+frame_id: '' and grasped_object_mask header seq: 120967
+stamp:
+  secs: 1734488631
+  nsecs: 145029068
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is -4658393510 for nearest: 1734488626719776027 - target: 1734488631378169537 at idx 249
+[INFO] [1734488631.598122]: received mask_obs with header seq: 1643
+stamp:
+  secs: 1734488631
+  nsecs: 602123737
+frame_id: '' and grasped_object_mask header seq: 120973
+stamp:
+  secs: 1734488631
+  nsecs: 344864368
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488631.636867]: Storing episode...
+For topic gripper_width: timestamp difference is -4858229287 for nearest: 1734488626719776027 - target: 1734488631578005314 at idx 249
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1734488631.801016]: received mask_obs with header seq: 1644
+stamp:
+  secs: 1734488631
+  nsecs: 804388046
+frame_id: '' and grasped_object_mask header seq: 120979
+stamp:
+  secs: 1734488631
+  nsecs: 544699192
+frame_id: "camera_color_optical_frame"
+[INFO] [1734488632.431817]: Stored episode 3.
+[INFO] [1734488632.433711]: Saving video...
+[INFO] [1734488632.874029]: Video saved!
+Starting episode 3
+[INFO] [1734488632.902666]: resetting environment
+[INFO] [1734488632.909836]: cleared current plan
+[INFO] [1734488632.918970]: moving to home
+[INFO] [1734488635.623441]: reached home
+[INFO] [1734488635.634476]: reset action history
+[INFO] [1734488638.427099]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+program exiting gracefully
+proceeding to start episode!
+[INFO] [1734488722.939590]: Saving video...
+[INFO] [1734488722.954511]: Video saved!

212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt

@@ -0,0 +1,339 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.2
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2024-12-18T02:21:24.806252",
+    "startedAt": "2024-12-18T02:21:24.423927",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "5f28b54373d7d41e4ebace380ff1080a8ffc4051"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3375.3751250000005,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4501.58,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.929,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 389.73836517333984
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}

212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "7012418043e20ba472ca099eb2fd4bae9cf951ff5f200bf3b8b28bef7c455f17", "size": 860246, "path": "media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4"}, "global_step": 182, "_timestamp": 1734488600.9750466, "_runtime": 116.54287362098694, "_step": 5, "eval/1_eval": {"_type": "video-file", "sha256": "4a30824235f8d3481acdd75328ce66a4db21dec3ebd5b875ba7e3c527508d0f3", "size": 854722, "path": "media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4"}, "eval/num_success": 0.0, "episode": 2.0, "eval/success_rate": 0.0, "eval/2_eval": {"_type": "video-file", "sha256": "fe502a2ffcc54aa7653539234f578d5a1a01cd0bf111c0cb8107c59433150318", "size": 1236066, "path": "media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4"}, "eval/video": {"_type": "video-file", "sha256": "25445a41b5bbda0ed7e2e0845eaa5c96373705c5c738fe640c1ea70a07b4176c", "size": 262, "path": "media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4"}, "_wandb": {"runtime": 238}}

212118/wandb/run-20241217_212124-78r9r705/logs/debug-internal.log

@@ -0,0 +1,651 @@
+2024-12-17 21:21:24,431 INFO    StreamThr :76461 [internal.py:wandb_internal():85] W&B internal server running at pid: 76461, started at: 2024-12-17 21:21:24.431534
+2024-12-17 21:21:24,433 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status
+2024-12-17 21:21:24,444 INFO    WriterThread:76461 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb
+2024-12-17 21:21:24,444 DEBUG   SenderThread:76461 [sender.py:send():379] send: header
+2024-12-17 21:21:24,453 DEBUG   SenderThread:76461 [sender.py:send():379] send: run
+2024-12-17 21:21:24,691 INFO    SenderThread:76461 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files
+2024-12-17 21:21:24,691 INFO    SenderThread:76461 [sender.py:_start_run_threads():1188] run started: 78r9r705 with start time 1734488484.432173
+2024-12-17 21:21:24,705 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 21:21:24,706 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: check_version
+2024-12-17 21:21:24,787 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 21:21:24,792 DEBUG   HandlerThread:76461 [system_info.py:__init__():26] System info init
+2024-12-17 21:21:24,792 DEBUG   HandlerThread:76461 [system_info.py:__init__():41] System info init done
+2024-12-17 21:21:24,792 INFO    HandlerThread:76461 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 21:21:24,792 INFO    SystemMonitor:76461 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 21:21:24,792 INFO    HandlerThread:76461 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 21:21:24,793 INFO    SystemMonitor:76461 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 21:21:24,793 INFO    SystemMonitor:76461 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 21:21:24,793 INFO    SystemMonitor:76461 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 21:21:24,794 INFO    SystemMonitor:76461 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 21:21:24,794 INFO    SystemMonitor:76461 [interfaces.py:start():188] Started network monitoring
+2024-12-17 21:21:24,806 DEBUG   HandlerThread:76461 [system_info.py:probe():152] Probing system
+2024-12-17 21:21:24,807 DEBUG   HandlerThread:76461 [system_info.py:_probe_git():137] Probing git
+2024-12-17 21:21:24,812 DEBUG   HandlerThread:76461 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 21:21:24,812 DEBUG   HandlerThread:76461 [system_info.py:probe():200] Probing system done
+2024-12-17 21:21:24,812 DEBUG   HandlerThread:76461 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T02:21:24.806252', 'startedAt': '2024-12-18T02:21:24.423927', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3375.3751250000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4501.58, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.929, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 389.73836517333984}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 21:21:24,812 INFO    HandlerThread:76461 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 21:21:24,812 INFO    HandlerThread:76461 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 21:21:24,812 DEBUG   HandlerThread:76461 [system_info.py:_save_code():44] Saving code
+2024-12-17 21:21:24,814 DEBUG   HandlerThread:76461 [system_info.py:_save_code():67] Saving code done
+2024-12-17 21:21:24,814 DEBUG   HandlerThread:76461 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 21:21:24,883 DEBUG   HandlerThread:76461 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 21:21:24,883 INFO    HandlerThread:76461 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 21:21:24,886 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:21:24,886 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 21:21:24,887 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 21:21:24,887 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 21:21:25,126 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 21:21:25,126 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 21:21:25,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:25,128 DEBUG   SenderThread:76461 [sender.py:send():379] send: telemetry
+2024-12-17 21:21:25,133 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/zf5gfm67-diff.patch
+2024-12-17 21:21:25,157 INFO    wandb-upload_1:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/bk0z0q6f-code/FISH/eval_robot.py
+2024-12-17 21:21:25,159 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/o7jx2utn-wandb-metadata.json
+2024-12-17 21:21:25,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:21:25,169 INFO    HandlerThread:76461 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/tb"
+  save: true
+}
+control {
+  relay_id: "52ec4a1767a3429cb6d1b8e86029eaaf"
+}
+_info {
+  stream_id: "78r9r705"
+}
+
+2024-12-17 21:21:25,169 DEBUG   HandlerThread:76461 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 21:21:25,170 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:21:25,277 DEBUG   SenderThread:76461 [sender.py:send():379] send: tbrecord
+2024-12-17 21:21:25,278 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:21:25,278 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 with policy live
+2024-12-17 21:21:25,454 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/lobsc1b3-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch
+2024-12-17 21:21:25,693 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:25,694 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH
+2024-12-17 21:21:25,694 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code
+2024-12-17 21:21:26,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:27,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:27,692 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:28,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:29,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:30,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:30,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:31,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:31,696 DEBUG   SenderThread:76461 [sender.py:send():379] send: config
+2024-12-17 21:21:32,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:33,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:33,694 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:34,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:35,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:35,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:36,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:37,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:37,695 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:38,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:39,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:40,126 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:21:40,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:21:40,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:40,281 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:41,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:41,696 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:42,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:43,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:44,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:45,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:46,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:46,130 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:47,130 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:48,130 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:49,130 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:50,130 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:51,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:51,699 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:52,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:52,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:53,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:53,699 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:54,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:55,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:21:55,129 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:21:55,173 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:55,700 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:56,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:57,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:57,282 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:21:57,701 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml
+2024-12-17 21:21:57,701 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:21:58,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:59,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:21:59,701 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:00,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:01,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:01,702 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:02,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:02,562 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:03,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:03,702 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:04,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:05,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:05,703 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:06,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:07,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:07,703 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:07,736 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:08,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:09,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:09,704 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:10,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:22:10,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:22:10,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:11,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:11,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:12,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:12,599 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:22:12,599 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 with policy now
+2024-12-17 21:22:12,599 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:22:12,600 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:22:12,600 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:22:12,601 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:22:12,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4
+2024-12-17 21:22:12,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:22:12,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval
+2024-12-17 21:22:12,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media
+2024-12-17 21:22:12,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos
+2024-12-17 21:22:12,849 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/0wiijtor-media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4
+2024-12-17 21:22:13,108 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/2ltzk93y-media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4
+2024-12-17 21:22:13,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:13,611 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:13,705 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:14,138 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:15,134 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:16,134 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:17,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:17,707 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:18,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:18,625 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:19,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:19,707 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:20,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:21,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:22,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:23,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:24,129 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:24,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:24,794 DEBUG   SystemMonitor:76461 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 21:22:24,795 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:22:25,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:22:25,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:22:25,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:26,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:27,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:28,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:29,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:29,217 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:30,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:31,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:32,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:33,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:34,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:35,128 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:35,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:36,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:37,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:37,713 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:22:37,978 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/lzwjgswx-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:22:38,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:39,137 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:39,714 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:40,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:22:40,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:22:40,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:40,228 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:41,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:41,715 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:42,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:43,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:43,715 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:44,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:45,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:45,252 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:45,716 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:46,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:47,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:47,717 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:48,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:49,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:49,718 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:50,140 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:50,256 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:51,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:51,718 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:52,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:53,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:53,719 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:54,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:54,796 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:22:55,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:22:55,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:22:55,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:55,267 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:22:55,720 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:56,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:57,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:57,720 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:22:58,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:59,141 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:22:59,721 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:00,142 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:00,373 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:00,417 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:23:00,417 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:23:00,418 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 with policy now
+2024-12-17 21:23:00,418 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:23:00,418 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:23:00,418 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:23:00,673 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/4e5seuc5-media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4
+2024-12-17 21:23:00,721 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:23:00,722 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4
+2024-12-17 21:23:00,722 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval
+2024-12-17 21:23:00,967 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/akortq61-media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4
+2024-12-17 21:23:01,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:01,722 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:02,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:03,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:04,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:05,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:05,723 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:06,135 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:06,160 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:07,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:07,724 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:08,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:09,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:10,131 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:23:10,131 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:23:10,173 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:11,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:11,232 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:12,146 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:13,146 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:14,146 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:15,146 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:16,146 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:17,133 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:17,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:18,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:19,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:20,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:21,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:21,571 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:23:21,571 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:23:21,571 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:23:21,729 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:23:21,729 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:23:21,729 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:22,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:23,029 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:23,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:23,729 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:24,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:24,796 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:23:25,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:23:25,128 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:23:25,147 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:25,730 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:26,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:27,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:27,731 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:28,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:28,194 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:29,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:29,731 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:30,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:31,148 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:31,732 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:32,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:33,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:33,393 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:33,733 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:34,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:35,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:35,734 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:36,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:37,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:37,734 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:38,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:38,405 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:39,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:39,735 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:40,127 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:23:40,127 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:23:40,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:41,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:41,736 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:42,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:43,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:43,409 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:43,737 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:44,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:45,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:45,737 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:46,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:47,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:47,738 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:48,151 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:48,460 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:49,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:49,739 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:50,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:51,151 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:51,740 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:52,150 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:52,900 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:23:52,900 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 with policy now
+2024-12-17 21:23:52,900 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:23:52,901 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:23:52,901 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:23:52,901 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:23:53,155 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/evzpq5tf-media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4
+2024-12-17 21:23:53,155 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:53,741 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:23:53,741 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:53,741 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4
+2024-12-17 21:23:53,741 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval
+2024-12-17 21:23:53,920 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:54,016 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/jsf3d998-media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4
+2024-12-17 21:23:54,163 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:54,797 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:23:55,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:23:55,133 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:23:55,177 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:56,161 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:57,159 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:57,742 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:23:58,163 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:59,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:23:59,469 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:23:59,743 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:24:00,165 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:01,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:02,171 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:03,174 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:04,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:05,144 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:05,169 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:06,176 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:07,167 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:08,176 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:09,171 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:10,136 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:24:10,136 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:24:10,177 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:10,211 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:11,171 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:12,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:13,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:14,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:15,174 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:16,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:16,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:17,182 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:18,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:19,182 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:20,177 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:21,183 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:22,145 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:22,178 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:23,186 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:24,181 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:24,797 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:24:25,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:24:25,133 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:24:25,185 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:26,179 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:27,184 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:27,208 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:28,189 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:29,188 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:30,185 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:31,190 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:32,189 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:33,152 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:33,187 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:34,191 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:35,190 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:36,191 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:37,190 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:38,191 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:39,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:39,200 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:40,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:24:40,132 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:24:40,201 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:41,203 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:42,204 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:43,205 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:44,203 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:44,227 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:45,211 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:46,206 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:47,208 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:48,214 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:49,209 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:50,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:50,210 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:51,210 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:52,211 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:53,210 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:54,214 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:54,798 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:24:55,132 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:24:55,133 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:24:55,192 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:24:55,213 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:56,214 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:57,210 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:58,213 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:24:59,220 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:00,223 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:01,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:01,219 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:02,216 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:03,216 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:04,218 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:05,220 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:06,226 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:07,149 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:07,223 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:08,228 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:09,225 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:10,142 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:25:10,142 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:25:10,237 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:11,233 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:11,766 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:25:12,227 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:12,264 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:13,237 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:14,236 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:15,256 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:16,257 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:17,264 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:18,160 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:18,246 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:19,253 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:20,253 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:21,248 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:22,252 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:23,047 DEBUG   SenderThread:76461 [sender.py:send():379] send: files
+2024-12-17 21:25:23,047 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 with policy now
+2024-12-17 21:25:23,047 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:25:23,047 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:25:23,048 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:25:23,049 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:25:23,051 DEBUG   SenderThread:76461 [sender.py:send():379] send: exit
+2024-12-17 21:25:23,051 INFO    SenderThread:76461 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 21:25:23,051 INFO    SenderThread:76461 [sender.py:send_exit():588] handling runtime: 238
+2024-12-17 21:25:23,051 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:25:23,051 INFO    SenderThread:76461 [sender.py:send_exit():594] send defer
+2024-12-17 21:25:23,051 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:23,051 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 21:25:23,052 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:23,052 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 21:25:23,052 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 1
+2024-12-17 21:25:23,052 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:23,052 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 21:25:23,052 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:23,052 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 21:25:23,052 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 2
+2024-12-17 21:25:23,052 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:23,052 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 21:25:23,052 INFO    HandlerThread:76461 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 21:25:23,052 DEBUG   SystemMonitor:76461 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 21:25:23,052 INFO    HandlerThread:76461 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 21:25:23,052 DEBUG   SystemMonitor:76461 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 21:25:23,053 INFO    HandlerThread:76461 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 21:25:23,096 INFO    HandlerThread:76461 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 21:25:23,096 INFO    HandlerThread:76461 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 21:25:23,096 INFO    HandlerThread:76461 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 21:25:23,096 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:23,096 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 21:25:23,096 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 3
+2024-12-17 21:25:23,097 DEBUG   SenderThread:76461 [sender.py:send():379] send: stats
+2024-12-17 21:25:23,097 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:23,097 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 21:25:23,097 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:23,097 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 21:25:23,097 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 4
+2024-12-17 21:25:23,097 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:23,097 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 21:25:23,269 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/rrl4ps4z-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4
+2024-12-17 21:25:23,770 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:25:23,770 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:25:23,770 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4
+2024-12-17 21:25:23,771 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval
+2024-12-17 21:25:24,015 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/ipbifp7e-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4
+2024-12-17 21:25:28,586 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:28,586 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:28,586 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:28,587 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 21:25:28,587 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 5
+2024-12-17 21:25:28,587 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:28,587 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:28,587 DEBUG   SenderThread:76461 [sender.py:send():379] send: history
+2024-12-17 21:25:28,587 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 21:25:28,587 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:25:28,588 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:25:28,588 DEBUG   SenderThread:76461 [sender.py:send():379] send: summary
+2024-12-17 21:25:28,588 INFO    SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:25:28,588 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:28,588 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 21:25:28,588 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 6
+2024-12-17 21:25:28,588 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:28,588 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 21:25:28,589 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:28,589 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 21:25:28,589 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 7
+2024-12-17 21:25:28,589 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:25:28,589 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:28,589 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 21:25:28,589 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:28,589 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 21:25:28,772 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:25:29,052 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 21:25:29,052 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:29,669 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 8
+2024-12-17 21:25:29,669 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:29,669 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:29,669 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 21:25:29,670 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:29,670 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 21:25:29,670 INFO    SenderThread:76461 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 21:25:29,672 INFO    SenderThread:76461 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 21:25:29,686 INFO    SenderThread:76461 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 21:25:29,700 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 9
+2024-12-17 21:25:29,700 DEBUG   SenderThread:76461 [sender.py:send():379] send: artifact
+2024-12-17 21:25:29,700 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:29,702 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 21:25:29,772 INFO    Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:25:30,053 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:30,189 INFO    wandb-upload_1:76461 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpat7ok6vh
+2024-12-17 21:25:30,320 INFO    wandb-upload_2:76461 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu793frvf
+2024-12-17 21:25:30,360 INFO    wandb-upload_0:76461 [upload_job.py:push():88] Uploaded file /tmp/tmp41i79nsx/wandb-job.json
+2024-12-17 21:25:30,913 INFO    SenderThread:76461 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzIyNjY3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzE5ODcyNA==', 'versionIndex': 378}}}
+2024-12-17 21:25:30,913 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:30,913 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 21:25:30,913 INFO    SenderThread:76461 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 21:25:31,773 INFO    SenderThread:76461 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files
+2024-12-17 21:25:31,773 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml config.yaml
+2024-12-17 21:25:31,773 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json wandb-summary.json
+2024-12-17 21:25:31,773 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:25:31,774 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt requirements.txt
+2024-12-17 21:25:31,774 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 21:25:31,774 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log output.log
+2024-12-17 21:25:31,774 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch diff.patch
+2024-12-17 21:25:31,777 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 21:25:31,777 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4
+2024-12-17 21:25:31,777 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4
+2024-12-17 21:25:31,778 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4
+2024-12-17 21:25:31,779 INFO    SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4
+2024-12-17 21:25:31,780 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 10
+2024-12-17 21:25:31,780 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:31,782 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:31,782 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 21:25:31,782 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:31,782 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 21:25:31,782 INFO    SenderThread:76461 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:25:31,949 INFO    wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json
+2024-12-17 21:25:31,988 INFO    wandb-upload_1:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml
+2024-12-17 21:25:31,990 INFO    wandb-upload_3:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt
+2024-12-17 21:25:32,014 INFO    wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/kg0vdiiy-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0
+2024-12-17 21:25:32,036 INFO    wandb-upload_4:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log
+2024-12-17 21:25:32,053 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:32,054 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:32,237 INFO    Thread-11 (_thread_body):76461 [sender.py:transition_state():614] send defer: 11
+2024-12-17 21:25:32,237 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:32,237 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 21:25:32,237 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:32,237 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 21:25:32,237 INFO    SenderThread:76461 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 21:25:32,238 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 12
+2024-12-17 21:25:32,238 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:32,238 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 21:25:32,238 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:32,238 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 21:25:32,238 INFO    SenderThread:76461 [file_stream.py:finish():601] file stream finish called
+2024-12-17 21:25:32,430 INFO    SenderThread:76461 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 21:25:32,430 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 13
+2024-12-17 21:25:32,430 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:32,430 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 21:25:32,430 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:32,430 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 21:25:32,430 INFO    SenderThread:76461 [sender.py:transition_state():614] send defer: 14
+2024-12-17 21:25:32,430 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:25:32,431 INFO    HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 21:25:32,431 DEBUG   SenderThread:76461 [sender.py:send():379] send: final
+2024-12-17 21:25:32,431 DEBUG   SenderThread:76461 [sender.py:send():379] send: footer
+2024-12-17 21:25:32,431 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:25:32,431 INFO    SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 21:25:32,431 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:32,431 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 21:25:32,432 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:25:32,432 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:32,432 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: server_info
+2024-12-17 21:25:32,432 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 21:25:32,434 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 21:25:32,434 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:25:32,507 DEBUG   SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:25:32,507 INFO    MainThread:76461 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 21:25:32,507 INFO    MainThread:76461 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 21:25:32,507 INFO    MainThread:76461 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 21:25:32,507 DEBUG   HandlerThread:76461 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 21:25:32,507 INFO    HandlerThread:76461 [handler.py:finish():882] shutting down handler
+2024-12-17 21:25:33,432 INFO    WriterThread:76461 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb
+2024-12-17 21:25:33,507 INFO    SenderThread:76461 [sender.py:finish():1615] shutting down sender
+2024-12-17 21:25:33,507 INFO    SenderThread:76461 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:25:33,507 INFO    SenderThread:76461 [file_pusher.py:join():175] waiting for file pusher

212118/wandb/run-20241217_212124-78r9r705/logs/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 21:21:24,425 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 21:21:24,425 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Configure stats pid to 76272
+2024-12-17 21:21:24,425 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/settings
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/logs/debug.log
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/logs/debug-internal.log
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:init():569] calling init triggers
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '523_0', 'wandb_notes': '523_0_req_603_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['fowlers_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118'}
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:init():619] starting backend
+2024-12-17 21:21:24,426 INFO    MainThread:76272 [wandb_init.py:init():623] setting up manager
+2024-12-17 21:21:24,430 INFO    MainThread:76272 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 21:21:24,431 INFO    MainThread:76272 [wandb_init.py:init():631] backend started and connected
+2024-12-17 21:21:24,443 INFO    MainThread:76272 [wandb_init.py:init():720] updated telemetry
+2024-12-17 21:21:24,450 INFO    MainThread:76272 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 21:21:24,705 INFO    MainThread:76272 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 21:21:24,776 INFO    MainThread:76272 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 21:21:24,779 INFO    MainThread:76272 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 21:21:25,127 INFO    MainThread:76272 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 21:21:25,127 INFO    MainThread:76272 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 21:21:25,127 INFO    MainThread:76272 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 21:21:25,127 INFO    MainThread:76272 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 21:21:25,129 INFO    MainThread:76272 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 21:21:25,129 INFO    MainThread:76272 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/tb, True
+2024-12-17 21:21:31,696 INFO    MainThread:76272 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'}
+2024-12-17 21:25:33,510 WARNING MsgRouterThr:76272 [router.py:message_loop():77] message_loop has been closed

212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d081babd0cb3a79eceffd7c8022f97e7e0cbbc6ef1dc6f783b0b53f328279a13
+size 159910

215239/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '523_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

215239/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215239
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

215239/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

215239/eval_policy.log

@@ -0,0 +1,15 @@
+[2024-12-17 21:52:39,536][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 21:52:39,540][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-17 21:52:41,548][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

215240/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '523_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

215240/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

215240/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

215240/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

215240/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

215240/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

215240/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

215240/episode_rosbags/episode_0_2024-12-17-21-53-24.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:327377ba0f4c35a511dcbba69c5edef66fb7bf0f9f47b8b143509addeb82a43c
+size 1410388741

215240/episode_rosbags/episode_1_2024-12-17-21-54-05.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a3957ac7f81c2697bc53a35e8836a90828121ea8cc68cf2bb77f09f9fa2beb2
+size 1407810142